ia64/xen-unstable

view xen/arch/x86/smpboot.c @ 5374:22e42640bcff

bitkeeper revision 1.1691.1.8 (42a6fb21d3oJwpLmOxa2jKHRJ-8fJg)

First phase of removing IRQ numbers from Xen (transitioning to
IRQ addressing by 'legacy ISA IRQ', 'interrupt vector', and
'I/O APIC address + pin' as appropriate). Overall plan is to move
I/O APIC parsing and setup out of Xen (so we start DOM0 in virtual wire
mode).
Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Wed Jun 08 14:05:21 2005 +0000 (2005-06-08)
parents 8651a99cdc09
children 949970efef98
line source
1 /*
2 * x86 SMP booting functions
3 *
4 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
5 * (c) 1998, 1999, 2000 Ingo Molnar <mingo@redhat.com>
6 *
7 * Much of the core SMP work is based on previous work by Thomas Radke, to
8 * whom a great many thanks are extended.
9 *
10 * Thanks to Intel for making available several different Pentium,
11 * Pentium Pro and Pentium-II/Xeon MP machines.
12 * Original development of Linux SMP code supported by Caldera.
13 *
14 * This code is released under the GNU General Public License version 2 or
15 * later.
16 *
17 * Fixes
18 * Felix Koop : NR_CPUS used properly
19 * Jose Renau : Handle single CPU case.
20 * Alan Cox : By repeated request 8) - Total BogoMIPS report.
21 * Greg Wright : Fix for kernel stacks panic.
22 * Erich Boleyn : MP v1.4 and additional changes.
23 * Matthias Sattler : Changes for 2.1 kernel map.
24 * Michel Lespinasse : Changes for 2.1 kernel map.
25 * Michael Chastain : Change trampoline.S to gnu as.
26 * Alan Cox : Dumb bug: 'B' step PPro's are fine
27 * Ingo Molnar : Added APIC timers, based on code
28 * from Jose Renau
29 * Ingo Molnar : various cleanups and rewrites
30 * Tigran Aivazian : fixed "0.00 in /proc/uptime on SMP" bug.
31 * Maciej W. Rozycki : Bits for genuine 82489DX APICs
32 * Martin J. Bligh : Added support for multi-quad systems
33 * Dave Jones : Report invalid combinations of Athlon CPUs.
34 * Rusty Russell : Hacked into shape for new "hotplug" boot process. */
36 #include <xen/config.h>
37 #include <xen/init.h>
38 #include <xen/kernel.h>
39 #include <xen/mm.h>
40 #include <xen/sched.h>
41 #include <xen/irq.h>
42 #include <xen/delay.h>
43 #include <asm/current.h>
44 #include <asm/mc146818rtc.h>
45 #include <asm/desc.h>
46 #include <asm/div64.h>
47 #include <asm/flushtlb.h>
48 #include <asm/msr.h>
49 #include <mach_apic.h>
50 #include <mach_wakecpu.h>
52 static int _foo;
53 #define set_kernel_exec(x,y) (_foo=0)
54 #define alloc_bootmem_low_pages(x) __va(0x90000) /* trampoline address */
55 int tainted;
56 #define TAINT_UNSAFE_SMP 0
58 /* Set if we find a B stepping CPU */
59 static int __initdata smp_b_stepping;
61 /* Number of siblings per CPU package */
62 int smp_num_siblings = 1;
63 int phys_proc_id[NR_CPUS]; /* Package ID of each logical CPU */
64 EXPORT_SYMBOL(phys_proc_id);
66 /* bitmap of online cpus */
67 cpumask_t cpu_online_map;
69 cpumask_t cpu_callin_map;
70 cpumask_t cpu_callout_map;
71 static cpumask_t smp_commenced_mask;
73 /* Per CPU bogomips and other parameters */
74 struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned;
76 u8 x86_cpu_to_apicid[NR_CPUS] =
77 { [0 ... NR_CPUS-1] = 0xff };
78 EXPORT_SYMBOL(x86_cpu_to_apicid);
80 /*
81 * Trampoline 80x86 program as an array.
82 */
84 extern unsigned char trampoline_data [];
85 extern unsigned char trampoline_end [];
86 static unsigned char *trampoline_base;
87 static int trampoline_exec;
89 /*
90 * Currently trivial. Write the real->protected mode
91 * bootstrap into the page concerned. The caller
92 * has made sure it's suitably aligned.
93 */
95 static unsigned long __init setup_trampoline(void)
96 {
97 memcpy(trampoline_base, trampoline_data, trampoline_end - trampoline_data);
98 return virt_to_phys(trampoline_base);
99 }
101 /*
102 * We are called very early to get the low memory for the
103 * SMP bootup trampoline page.
104 */
105 void __init smp_alloc_memory(void)
106 {
107 trampoline_base = (void *) alloc_bootmem_low_pages(PAGE_SIZE);
108 /*
109 * Has to be in very low memory so we can execute
110 * real-mode AP code.
111 */
112 if (__pa(trampoline_base) >= 0x9F000)
113 BUG();
114 /*
115 * Make the SMP trampoline executable:
116 */
117 trampoline_exec = set_kernel_exec((unsigned long)trampoline_base, 1);
118 }
120 /*
121 * The bootstrap kernel entry code has set these up. Save them for
122 * a given CPU
123 */
125 static void __init smp_store_cpu_info(int id)
126 {
127 struct cpuinfo_x86 *c = cpu_data + id;
129 *c = boot_cpu_data;
130 if (id!=0)
131 identify_cpu(c);
132 /*
133 * Mask B, Pentium, but not Pentium MMX
134 */
135 if (c->x86_vendor == X86_VENDOR_INTEL &&
136 c->x86 == 5 &&
137 c->x86_mask >= 1 && c->x86_mask <= 4 &&
138 c->x86_model <= 3)
139 /*
140 * Remember we have B step Pentia with bugs
141 */
142 smp_b_stepping = 1;
144 /*
145 * Certain Athlons might work (for various values of 'work') in SMP
146 * but they are not certified as MP capable.
147 */
148 if ((c->x86_vendor == X86_VENDOR_AMD) && (c->x86 == 6)) {
150 /* Athlon 660/661 is valid. */
151 if ((c->x86_model==6) && ((c->x86_mask==0) || (c->x86_mask==1)))
152 goto valid_k7;
154 /* Duron 670 is valid */
155 if ((c->x86_model==7) && (c->x86_mask==0))
156 goto valid_k7;
158 /*
159 * Athlon 662, Duron 671, and Athlon >model 7 have capability bit.
160 * It's worth noting that the A5 stepping (662) of some Athlon XP's
161 * have the MP bit set.
162 * See http://www.heise.de/newsticker/data/jow-18.10.01-000 for more.
163 */
164 if (((c->x86_model==6) && (c->x86_mask>=2)) ||
165 ((c->x86_model==7) && (c->x86_mask>=1)) ||
166 (c->x86_model> 7))
167 if (cpu_has_mp)
168 goto valid_k7;
170 /* If we get here, it's not a certified SMP capable AMD system. */
171 tainted |= TAINT_UNSAFE_SMP;
172 }
174 valid_k7:
175 ;
176 }
178 /*
179 * TSC synchronization.
180 *
181 * We first check whether all CPUs have their TSC's synchronized,
182 * then we print a warning if not, and always resync.
183 */
185 static atomic_t tsc_start_flag = ATOMIC_INIT(0);
186 static atomic_t tsc_count_start = ATOMIC_INIT(0);
187 static atomic_t tsc_count_stop = ATOMIC_INIT(0);
188 static unsigned long long tsc_values[NR_CPUS];
190 #define NR_LOOPS 5
192 static void __init synchronize_tsc_bp (void)
193 {
194 int i;
195 unsigned long long t0;
196 unsigned long long sum, avg;
197 long long delta;
198 unsigned long one_usec;
199 int buggy = 0;
201 printk(KERN_INFO "checking TSC synchronization across %u CPUs: ", num_booting_cpus());
203 /* convert from kcyc/sec to cyc/usec */
204 one_usec = cpu_khz / 1000;
206 atomic_set(&tsc_start_flag, 1);
207 wmb();
209 /*
210 * We loop a few times to get a primed instruction cache,
211 * then the last pass is more or less synchronized and
212 * the BP and APs set their cycle counters to zero all at
213 * once. This reduces the chance of having random offsets
214 * between the processors, and guarantees that the maximum
215 * delay between the cycle counters is never bigger than
216 * the latency of information-passing (cachelines) between
217 * two CPUs.
218 */
219 for (i = 0; i < NR_LOOPS; i++) {
220 /*
221 * all APs synchronize but they loop on '== num_cpus'
222 */
223 while (atomic_read(&tsc_count_start) != num_booting_cpus()-1)
224 mb();
225 atomic_set(&tsc_count_stop, 0);
226 wmb();
227 /*
228 * this lets the APs save their current TSC:
229 */
230 atomic_inc(&tsc_count_start);
232 rdtscll(tsc_values[smp_processor_id()]);
233 /*
234 * We clear the TSC in the last loop:
235 */
236 if (i == NR_LOOPS-1)
237 write_tsc(0, 0);
239 /*
240 * Wait for all APs to leave the synchronization point:
241 */
242 while (atomic_read(&tsc_count_stop) != num_booting_cpus()-1)
243 mb();
244 atomic_set(&tsc_count_start, 0);
245 wmb();
246 atomic_inc(&tsc_count_stop);
247 }
249 sum = 0;
250 for (i = 0; i < NR_CPUS; i++) {
251 if (cpu_isset(i, cpu_callout_map)) {
252 t0 = tsc_values[i];
253 sum += t0;
254 }
255 }
256 avg = sum;
257 do_div(avg, num_booting_cpus());
259 sum = 0;
260 for (i = 0; i < NR_CPUS; i++) {
261 if (!cpu_isset(i, cpu_callout_map))
262 continue;
263 delta = tsc_values[i] - avg;
264 if (delta < 0)
265 delta = -delta;
266 /*
267 * We report bigger than 2 microseconds clock differences.
268 */
269 if (delta > 2*one_usec) {
270 long realdelta;
271 if (!buggy) {
272 buggy = 1;
273 printk("\n");
274 }
275 realdelta = delta;
276 do_div(realdelta, one_usec);
277 if (tsc_values[i] < avg)
278 realdelta = -realdelta;
280 printk(KERN_INFO "CPU#%d had %ld usecs TSC skew, fixed it up.\n", i, realdelta);
281 }
283 sum += delta;
284 }
285 if (!buggy)
286 printk("passed.\n");
287 }
289 static void __init synchronize_tsc_ap (void)
290 {
291 int i;
293 /*
294 * Not every cpu is online at the time
295 * this gets called, so we first wait for the BP to
296 * finish SMP initialization:
297 */
298 while (!atomic_read(&tsc_start_flag)) mb();
300 for (i = 0; i < NR_LOOPS; i++) {
301 atomic_inc(&tsc_count_start);
302 while (atomic_read(&tsc_count_start) != num_booting_cpus())
303 mb();
305 rdtscll(tsc_values[smp_processor_id()]);
306 if (i == NR_LOOPS-1)
307 write_tsc(0, 0);
309 atomic_inc(&tsc_count_stop);
310 while (atomic_read(&tsc_count_stop) != num_booting_cpus()) mb();
311 }
312 }
313 #undef NR_LOOPS
315 extern void calibrate_delay(void);
317 static atomic_t init_deasserted;
319 void __init smp_callin(void)
320 {
321 int cpuid, phys_id, i;
323 /*
324 * If waken up by an INIT in an 82489DX configuration
325 * we may get here before an INIT-deassert IPI reaches
326 * our local APIC. We have to wait for the IPI or we'll
327 * lock up on an APIC access.
328 */
329 wait_for_init_deassert(&init_deasserted);
331 /*
332 * (This works even if the APIC is not enabled.)
333 */
334 phys_id = GET_APIC_ID(apic_read(APIC_ID));
335 cpuid = smp_processor_id();
336 if (cpu_isset(cpuid, cpu_callin_map)) {
337 printk("huh, phys CPU#%d, CPU#%d already present??\n",
338 phys_id, cpuid);
339 BUG();
340 }
341 Dprintk("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
343 /*
344 * STARTUP IPIs are fragile beasts as they might sometimes
345 * trigger some glue motherboard logic. Complete APIC bus
346 * silence for 1 second, this overestimates the time the
347 * boot CPU is spending to send the up to 2 STARTUP IPIs
348 * by a factor of two. This should be enough.
349 */
351 /*
352 * Waiting 2s total for startup
353 */
354 for (i = 0; i < 200; i++) {
355 /*
356 * Has the boot CPU finished it's STARTUP sequence?
357 */
358 if (cpu_isset(cpuid, cpu_callout_map))
359 break;
360 rep_nop();
361 mdelay(10);
362 }
364 if (!cpu_isset(cpuid, cpu_callout_map)) {
365 printk("BUG: CPU%d started up but did not get a callout!\n",
366 cpuid);
367 BUG();
368 }
370 /*
371 * the boot CPU has finished the init stage and is spinning
372 * on callin_map until we finish. We are free to set up this
373 * CPU, first the APIC. (this is probably redundant on most
374 * boards)
375 */
377 Dprintk("CALLIN, before setup_local_APIC().\n");
378 smp_callin_clear_local_apic();
379 setup_local_APIC();
380 map_cpu_to_logical_apicid();
382 #if 0
383 /*
384 * Get our bogomips.
385 */
386 calibrate_delay();
387 Dprintk("Stack at about %p\n",&cpuid);
388 #endif
390 /*
391 * Save our processor parameters
392 */
393 smp_store_cpu_info(cpuid);
395 disable_APIC_timer();
397 /*
398 * Allow the master to continue.
399 */
400 cpu_set(cpuid, cpu_callin_map);
402 /*
403 * Synchronize the TSC with the BP
404 */
405 if (cpu_has_tsc && cpu_khz)
406 synchronize_tsc_ap();
407 }
409 int cpucount;
411 #ifdef CONFIG_X86_32
412 static void construct_percpu_idt(unsigned int cpu)
413 {
414 unsigned char idt_load[10];
416 idt_tables[cpu] = xmalloc_array(idt_entry_t, IDT_ENTRIES);
417 memcpy(idt_tables[cpu], idt_table, IDT_ENTRIES*sizeof(idt_entry_t));
419 *(unsigned short *)(&idt_load[0]) = (IDT_ENTRIES*sizeof(idt_entry_t))-1;
420 *(unsigned long *)(&idt_load[2]) = (unsigned long)idt_tables[cpu];
421 __asm__ __volatile__ ( "lidt %0" : "=m" (idt_load) );
422 }
423 #endif
425 /*
426 * Activate a secondary processor.
427 */
428 void __init start_secondary(void *unused)
429 {
430 unsigned int cpu = cpucount;
432 extern void percpu_traps_init(void);
433 extern void cpu_init(void);
435 set_current(idle_task[cpu]);
436 set_processor_id(cpu);
438 percpu_traps_init();
440 cpu_init();
441 smp_callin();
442 while (!cpu_isset(smp_processor_id(), smp_commenced_mask))
443 rep_nop();
445 #ifdef CONFIG_X86_32
446 /*
447 * At this point, boot CPU has fully initialised the IDT. It is
448 * now safe to make ourselves a private copy.
449 */
450 construct_percpu_idt(cpu);
451 #endif
453 setup_secondary_APIC_clock();
454 enable_APIC_timer();
456 /*
457 * low-memory mappings have been cleared, flush them from
458 * the local TLBs too.
459 */
460 local_flush_tlb();
461 cpu_set(smp_processor_id(), cpu_online_map);
463 /* We can take interrupts now: we're officially "up". */
464 local_irq_enable();
466 wmb();
467 startup_cpu_idle_loop();
468 }
470 extern struct {
471 void * esp;
472 unsigned short ss;
473 } stack_start;
475 #ifdef CONFIG_NUMA
477 /* which logical CPUs are on which nodes */
478 cpumask_t node_2_cpu_mask[MAX_NUMNODES] =
479 { [0 ... MAX_NUMNODES-1] = CPU_MASK_NONE };
480 /* which node each logical CPU is on */
481 int cpu_2_node[NR_CPUS] = { [0 ... NR_CPUS-1] = 0 };
482 EXPORT_SYMBOL(cpu_2_node);
484 /* set up a mapping between cpu and node. */
485 static inline void map_cpu_to_node(int cpu, int node)
486 {
487 printk("Mapping cpu %d to node %d\n", cpu, node);
488 cpu_set(cpu, node_2_cpu_mask[node]);
489 cpu_2_node[cpu] = node;
490 }
492 /* undo a mapping between cpu and node. */
493 static inline void unmap_cpu_to_node(int cpu)
494 {
495 int node;
497 printk("Unmapping cpu %d from all nodes\n", cpu);
498 for (node = 0; node < MAX_NUMNODES; node ++)
499 cpu_clear(cpu, node_2_cpu_mask[node]);
500 cpu_2_node[cpu] = 0;
501 }
502 #else /* !CONFIG_NUMA */
504 #define map_cpu_to_node(cpu, node) ({})
505 #define unmap_cpu_to_node(cpu) ({})
507 #endif /* CONFIG_NUMA */
509 u8 cpu_2_logical_apicid[NR_CPUS] = { [0 ... NR_CPUS-1] = BAD_APICID };
511 void map_cpu_to_logical_apicid(void)
512 {
513 int cpu = smp_processor_id();
514 int apicid = logical_smp_processor_id();
516 cpu_2_logical_apicid[cpu] = apicid;
517 map_cpu_to_node(cpu, apicid_to_node(apicid));
518 }
520 void unmap_cpu_to_logical_apicid(int cpu)
521 {
522 cpu_2_logical_apicid[cpu] = BAD_APICID;
523 unmap_cpu_to_node(cpu);
524 }
526 #if APIC_DEBUG
527 static inline void __inquire_remote_apic(int apicid)
528 {
529 int i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
530 char *names[] = { "ID", "VERSION", "SPIV" };
531 int timeout, status;
533 printk("Inquiring remote APIC #%d...\n", apicid);
535 for (i = 0; i < sizeof(regs) / sizeof(*regs); i++) {
536 printk("... APIC #%d %s: ", apicid, names[i]);
538 /*
539 * Wait for idle.
540 */
541 apic_wait_icr_idle();
543 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(apicid));
544 apic_write_around(APIC_ICR, APIC_DM_REMRD | regs[i]);
546 timeout = 0;
547 do {
548 udelay(100);
549 status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
550 } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
552 switch (status) {
553 case APIC_ICR_RR_VALID:
554 status = apic_read(APIC_RRR);
555 printk("%08x\n", status);
556 break;
557 default:
558 printk("failed\n");
559 }
560 }
561 }
562 #endif
564 #ifdef WAKE_SECONDARY_VIA_NMI
565 /*
566 * Poke the other CPU in the eye via NMI to wake it up. Remember that the normal
567 * INIT, INIT, STARTUP sequence will reset the chip hard for us, and this
568 * won't ... remember to clear down the APIC, etc later.
569 */
570 static int __init
571 wakeup_secondary_cpu(int logical_apicid, unsigned long start_eip)
572 {
573 unsigned long send_status = 0, accept_status = 0;
574 int timeout, maxlvt;
576 /* Target chip */
577 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(logical_apicid));
579 /* Boot on the stack */
580 /* Kick the second */
581 apic_write_around(APIC_ICR, APIC_DM_NMI | APIC_DEST_LOGICAL);
583 Dprintk("Waiting for send to finish...\n");
584 timeout = 0;
585 do {
586 Dprintk("+");
587 udelay(100);
588 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
589 } while (send_status && (timeout++ < 1000));
591 /*
592 * Give the other CPU some time to accept the IPI.
593 */
594 udelay(200);
595 /*
596 * Due to the Pentium erratum 3AP.
597 */
598 maxlvt = get_maxlvt();
599 if (maxlvt > 3) {
600 apic_read_around(APIC_SPIV);
601 apic_write(APIC_ESR, 0);
602 }
603 accept_status = (apic_read(APIC_ESR) & 0xEF);
604 Dprintk("NMI sent.\n");
606 if (send_status)
607 printk("APIC never delivered???\n");
608 if (accept_status)
609 printk("APIC delivery error (%lx).\n", accept_status);
611 return (send_status | accept_status);
612 }
613 #endif /* WAKE_SECONDARY_VIA_NMI */
615 #ifdef WAKE_SECONDARY_VIA_INIT
616 static int __init
617 wakeup_secondary_cpu(int phys_apicid, unsigned long start_eip)
618 {
619 unsigned long send_status = 0, accept_status = 0;
620 int maxlvt, timeout, num_starts, j;
622 /*
623 * Be paranoid about clearing APIC errors.
624 */
625 if (APIC_INTEGRATED(apic_version[phys_apicid])) {
626 apic_read_around(APIC_SPIV);
627 apic_write(APIC_ESR, 0);
628 apic_read(APIC_ESR);
629 }
631 Dprintk("Asserting INIT.\n");
633 /*
634 * Turn INIT on target chip
635 */
636 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
638 /*
639 * Send IPI
640 */
641 apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_INT_ASSERT
642 | APIC_DM_INIT);
644 Dprintk("Waiting for send to finish...\n");
645 timeout = 0;
646 do {
647 Dprintk("+");
648 udelay(100);
649 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
650 } while (send_status && (timeout++ < 1000));
652 mdelay(10);
654 Dprintk("Deasserting INIT.\n");
656 /* Target chip */
657 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
659 /* Send IPI */
660 apic_write_around(APIC_ICR, APIC_INT_LEVELTRIG | APIC_DM_INIT);
662 Dprintk("Waiting for send to finish...\n");
663 timeout = 0;
664 do {
665 Dprintk("+");
666 udelay(100);
667 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
668 } while (send_status && (timeout++ < 1000));
670 atomic_set(&init_deasserted, 1);
672 /*
673 * Should we send STARTUP IPIs ?
674 *
675 * Determine this based on the APIC version.
676 * If we don't have an integrated APIC, don't send the STARTUP IPIs.
677 */
678 if (APIC_INTEGRATED(apic_version[phys_apicid]))
679 num_starts = 2;
680 else
681 num_starts = 0;
683 /*
684 * Run STARTUP IPI loop.
685 */
686 Dprintk("#startup loops: %d.\n", num_starts);
688 maxlvt = get_maxlvt();
690 for (j = 1; j <= num_starts; j++) {
691 Dprintk("Sending STARTUP #%d.\n",j);
692 apic_read_around(APIC_SPIV);
693 apic_write(APIC_ESR, 0);
694 apic_read(APIC_ESR);
695 Dprintk("After apic_write.\n");
697 /*
698 * STARTUP IPI
699 */
701 /* Target chip */
702 apic_write_around(APIC_ICR2, SET_APIC_DEST_FIELD(phys_apicid));
704 /* Boot on the stack */
705 /* Kick the second */
706 apic_write_around(APIC_ICR, APIC_DM_STARTUP
707 | (start_eip >> 12));
709 /*
710 * Give the other CPU some time to accept the IPI.
711 */
712 udelay(300);
714 Dprintk("Startup point 1.\n");
716 Dprintk("Waiting for send to finish...\n");
717 timeout = 0;
718 do {
719 Dprintk("+");
720 udelay(100);
721 send_status = apic_read(APIC_ICR) & APIC_ICR_BUSY;
722 } while (send_status && (timeout++ < 1000));
724 /*
725 * Give the other CPU some time to accept the IPI.
726 */
727 udelay(200);
728 /*
729 * Due to the Pentium erratum 3AP.
730 */
731 if (maxlvt > 3) {
732 apic_read_around(APIC_SPIV);
733 apic_write(APIC_ESR, 0);
734 }
735 accept_status = (apic_read(APIC_ESR) & 0xEF);
736 if (send_status || accept_status)
737 break;
738 }
739 Dprintk("After Startup.\n");
741 if (send_status)
742 printk("APIC never delivered???\n");
743 if (accept_status)
744 printk("APIC delivery error (%lx).\n", accept_status);
746 return (send_status | accept_status);
747 }
748 #endif /* WAKE_SECONDARY_VIA_INIT */
750 extern cpumask_t cpu_initialized;
752 static int __init do_boot_cpu(int apicid)
753 /*
754 * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad
755 * (ie clustered apic addressing mode), this is a LOGICAL apic ID.
756 * Returns zero if CPU booted OK, else error code from wakeup_secondary_cpu.
757 */
758 {
759 struct domain *idle;
760 struct vcpu *v;
761 void *stack;
762 unsigned long boot_error;
763 int timeout, cpu;
764 unsigned long start_eip;
765 unsigned short nmi_high = 0, nmi_low = 0;
767 cpu = ++cpucount;
769 if ( (idle = do_createdomain(IDLE_DOMAIN_ID, cpu)) == NULL )
770 panic("failed 'createdomain' for CPU %d", cpu);
772 v = idle_task[cpu] = idle->vcpu[0];
774 set_bit(_DOMF_idle_domain, &idle->domain_flags);
776 v->arch.monitor_table = mk_pagetable(__pa(idle_pg_table));
778 /* start_eip had better be page-aligned! */
779 start_eip = setup_trampoline();
781 /* So we see what's up */
782 printk("Booting processor %d/%d eip %lx\n", cpu, apicid, start_eip);
784 stack = (void *)alloc_xenheap_pages(STACK_ORDER);
785 #if defined(__i386__)
786 stack_start.esp = (void *)__pa(stack);
787 #elif defined(__x86_64__)
788 stack_start.esp = stack;
789 #endif
790 stack_start.esp += STACK_SIZE - sizeof(struct cpu_info);
792 /* Debug build: detect stack overflow by setting up a guard page. */
793 memguard_guard_stack(stack);
795 /*
796 * This grunge runs the startup process for
797 * the targeted processor.
798 */
800 atomic_set(&init_deasserted, 0);
802 Dprintk("Setting warm reset code and vector.\n");
804 store_NMI_vector(&nmi_high, &nmi_low);
806 CMOS_WRITE(0xa, 0xf);
807 local_flush_tlb();
808 Dprintk("1.\n");
809 *((volatile unsigned short *) TRAMPOLINE_HIGH) = start_eip >> 4;
810 Dprintk("2.\n");
811 *((volatile unsigned short *) TRAMPOLINE_LOW) = start_eip & 0xf;
812 Dprintk("3.\n");
814 /*
815 * Starting actual IPI sequence...
816 */
817 boot_error = wakeup_secondary_cpu(apicid, start_eip);
819 if (!boot_error) {
820 /*
821 * allow APs to start initializing.
822 */
823 Dprintk("Before Callout %d.\n", cpu);
824 cpu_set(cpu, cpu_callout_map);
825 Dprintk("After Callout %d.\n", cpu);
827 /*
828 * Wait 5s total for a response
829 */
830 for (timeout = 0; timeout < 50000; timeout++) {
831 if (cpu_isset(cpu, cpu_callin_map))
832 break; /* It has booted */
833 udelay(100);
834 }
836 if (cpu_isset(cpu, cpu_callin_map)) {
837 /* number CPUs logically, starting from 1 (BSP is 0) */
838 Dprintk("OK.\n");
839 printk("CPU%d: ", cpu);
840 print_cpu_info(&cpu_data[cpu]);
841 Dprintk("CPU has booted.\n");
842 } else {
843 boot_error= 1;
844 if (*((volatile unsigned char *)trampoline_base)
845 == 0xA5)
846 /* trampoline started but...? */
847 printk("Stuck ??\n");
848 else
849 /* trampoline code not run */
850 printk("Not responding.\n");
851 inquire_remote_apic(apicid);
852 }
853 }
854 x86_cpu_to_apicid[cpu] = apicid;
855 if (boot_error) {
856 /* Try to put things back the way they were before ... */
857 unmap_cpu_to_logical_apicid(cpu);
858 cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
859 cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */
860 cpucount--;
861 }
863 /* mark "stuck" area as not stuck */
864 *((volatile unsigned long *)trampoline_base) = 0;
866 return boot_error;
867 }
869 #if 0
870 cycles_t cacheflush_time;
871 unsigned long cache_decay_ticks;
873 static void smp_tune_scheduling (void)
874 {
875 unsigned long cachesize; /* kB */
876 unsigned long bandwidth = 350; /* MB/s */
877 /*
878 * Rough estimation for SMP scheduling, this is the number of
879 * cycles it takes for a fully memory-limited process to flush
880 * the SMP-local cache.
881 *
882 * (For a P5 this pretty much means we will choose another idle
883 * CPU almost always at wakeup time (this is due to the small
884 * L1 cache), on PIIs it's around 50-100 usecs, depending on
885 * the cache size)
886 */
888 if (!cpu_khz) {
889 /*
890 * this basically disables processor-affinity
891 * scheduling on SMP without a TSC.
892 */
893 cacheflush_time = 0;
894 return;
895 } else {
896 cachesize = boot_cpu_data.x86_cache_size;
897 if (cachesize == -1) {
898 cachesize = 16; /* Pentiums, 2x8kB cache */
899 bandwidth = 100;
900 }
902 cacheflush_time = (cpu_khz>>10) * (cachesize<<10) / bandwidth;
903 }
905 cache_decay_ticks = (long)cacheflush_time/cpu_khz + 1;
907 printk("per-CPU timeslice cutoff: %ld.%02ld usecs.\n",
908 (long)cacheflush_time/(cpu_khz/1000),
909 ((long)cacheflush_time*100/(cpu_khz/1000)) % 100);
910 printk("task migration cache decay timeout: %ld msecs.\n",
911 cache_decay_ticks);
912 }
913 #else
914 #define smp_tune_scheduling() ((void)0)
915 #endif
917 /*
918 * Cycle through the processors sending APIC IPIs to boot each.
919 */
921 static int boot_cpu_logical_apicid;
922 /* Where the IO area was mapped on multiquad, always 0 otherwise */
923 void *xquad_portio;
925 cpumask_t cpu_sibling_map[NR_CPUS] __cacheline_aligned;
927 static void __init smp_boot_cpus(unsigned int max_cpus)
928 {
929 int apicid, cpu, bit, kicked;
930 #ifdef BOGOMIPS
931 unsigned long bogosum = 0;
932 #endif
934 /*
935 * Setup boot CPU information
936 */
937 smp_store_cpu_info(0); /* Final full version of the data */
938 printk("CPU%d: ", 0);
939 print_cpu_info(&cpu_data[0]);
941 boot_cpu_physical_apicid = GET_APIC_ID(apic_read(APIC_ID));
942 boot_cpu_logical_apicid = logical_smp_processor_id();
943 x86_cpu_to_apicid[0] = boot_cpu_physical_apicid;
945 /*current_thread_info()->cpu = 0;*/
946 smp_tune_scheduling();
947 cpus_clear(cpu_sibling_map[0]);
948 cpu_set(0, cpu_sibling_map[0]);
950 /*
951 * If we couldn't find an SMP configuration at boot time,
952 * get out of here now!
953 */
954 if (!smp_found_config && !acpi_lapic) {
955 printk(KERN_NOTICE "SMP motherboard not detected.\n");
956 init_uniprocessor:
957 phys_cpu_present_map = physid_mask_of_physid(0);
958 if (APIC_init_uniprocessor())
959 printk(KERN_NOTICE "Local APIC not detected."
960 " Using dummy APIC emulation.\n");
961 map_cpu_to_logical_apicid();
962 return;
963 }
965 /*
966 * Should not be necessary because the MP table should list the boot
967 * CPU too, but we do it for the sake of robustness anyway.
968 * Makes no sense to do this check in clustered apic mode, so skip it
969 */
970 if (!check_phys_apicid_present(boot_cpu_physical_apicid)) {
971 printk("weird, boot CPU (#%d) not listed by the BIOS.\n",
972 boot_cpu_physical_apicid);
973 physid_set(hard_smp_processor_id(), phys_cpu_present_map);
974 }
976 /*
977 * If we couldn't find a local APIC, then get out of here now!
978 */
979 if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) && !cpu_has_apic) {
980 printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n",
981 boot_cpu_physical_apicid);
982 goto init_uniprocessor;
983 }
985 verify_local_APIC();
987 /*
988 * If SMP should be disabled, then really disable it!
989 */
990 if (!max_cpus)
991 goto init_uniprocessor;
993 connect_bsp_APIC();
994 setup_local_APIC();
995 map_cpu_to_logical_apicid();
998 setup_portio_remap();
1000 /*
1001 * Scan the CPU present map and fire up the other CPUs via do_boot_cpu
1003 * In clustered apic mode, phys_cpu_present_map is a constructed thus:
1004 * bits 0-3 are quad0, 4-7 are quad1, etc. A perverse twist on the
1005 * clustered apic ID.
1006 */
1007 Dprintk("CPU present map: %lx\n", physids_coerce(phys_cpu_present_map));
1009 kicked = 1;
1010 for (bit = 0; kicked < NR_CPUS && bit < MAX_APICS; bit++) {
1011 apicid = cpu_present_to_apicid(bit);
1012 /*
1013 * Don't even attempt to start the boot CPU!
1014 */
1015 if ((apicid == boot_cpu_apicid) || (apicid == BAD_APICID))
1016 continue;
1018 if (!check_apicid_present(bit))
1019 continue;
1020 if (max_cpus <= cpucount+1)
1021 continue;
1023 if (do_boot_cpu(apicid))
1024 printk("CPU #%d not responding - cannot use it.\n",
1025 apicid);
1026 else
1027 ++kicked;
1030 /*
1031 * Install writable page 0 entry to set BIOS data area.
1032 */
1033 local_flush_tlb();
1035 /*
1036 * Paranoid: Set warm reset code and vector here back
1037 * to default values.
1038 */
1039 CMOS_WRITE(0, 0xf);
1041 *((volatile long *) phys_to_virt(0x467)) = 0;
1043 #ifdef BOGOMIPS
1044 /*
1045 * Allow the user to impress friends.
1046 */
1047 Dprintk("Before bogomips.\n");
1048 for (cpu = 0; cpu < NR_CPUS; cpu++)
1049 if (cpu_isset(cpu, cpu_callout_map))
1050 bogosum += cpu_data[cpu].loops_per_jiffy;
1051 printk(KERN_INFO
1052 "Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
1053 cpucount+1,
1054 bogosum/(500000/HZ),
1055 (bogosum/(5000/HZ))%100);
1056 #else
1057 printk("Total of %d processors activated.\n", cpucount+1);
1058 #endif
1060 Dprintk("Before bogocount - setting activated=1.\n");
1062 if (smp_b_stepping)
1063 printk(KERN_WARNING "WARNING: SMP operation may be unreliable with B stepping processors.\n");
1065 /*
1066 * Don't taint if we are running SMP kernel on a single non-MP
1067 * approved Athlon
1068 */
1069 if (tainted & TAINT_UNSAFE_SMP) {
1070 if (cpucount)
1071 printk (KERN_INFO "WARNING: This combination of AMD processors is not suitable for SMP.\n");
1072 else
1073 tainted &= ~TAINT_UNSAFE_SMP;
1076 Dprintk("Boot done.\n");
1078 /*
1079 * construct cpu_sibling_map[], so that we can tell sibling CPUs
1080 * efficiently.
1081 */
1082 for (cpu = 0; cpu < NR_CPUS; cpu++)
1083 cpus_clear(cpu_sibling_map[cpu]);
1085 for (cpu = 0; cpu < NR_CPUS; cpu++) {
1086 int siblings = 0;
1087 int i;
1088 if (!cpu_isset(cpu, cpu_callout_map))
1089 continue;
1091 if (smp_num_siblings > 1) {
1092 for (i = 0; i < NR_CPUS; i++) {
1093 if (!cpu_isset(i, cpu_callout_map))
1094 continue;
1095 if (phys_proc_id[cpu] == phys_proc_id[i]) {
1096 siblings++;
1097 cpu_set(i, cpu_sibling_map[cpu]);
1100 } else {
1101 siblings++;
1102 cpu_set(cpu, cpu_sibling_map[cpu]);
1105 if (siblings != smp_num_siblings)
1106 printk(KERN_WARNING "WARNING: %d siblings found for CPU%d, should be %d\n", siblings, cpu, smp_num_siblings);
1109 if (nmi_watchdog == NMI_LOCAL_APIC)
1110 check_nmi_watchdog();
1112 /*
1113 * Here we can be sure that there is an IO-APIC in the system. Let's
1114 * go and set it up:
1115 */
1116 if (!skip_ioapic_setup && nr_ioapics)
1117 setup_IO_APIC();
1119 setup_boot_APIC_clock();
1121 /*
1122 * Synchronize the TSC with the AP
1123 */
1124 if (cpu_has_tsc && cpucount && cpu_khz)
1125 synchronize_tsc_bp();
1128 /* These are wrappers to interface to the new boot process. Someone
1129 who understands all this stuff should rewrite it properly. --RR 15/Jul/02 */
1130 void __init smp_prepare_cpus(unsigned int max_cpus)
1132 smp_boot_cpus(max_cpus);
1135 void __devinit smp_prepare_boot_cpu(void)
1137 cpu_set(smp_processor_id(), cpu_online_map);
1138 cpu_set(smp_processor_id(), cpu_callout_map);
1141 int __devinit __cpu_up(unsigned int cpu)
1143 /* This only works at boot for x86. See "rewrite" above. */
1144 if (cpu_isset(cpu, smp_commenced_mask)) {
1145 local_irq_enable();
1146 return -ENOSYS;
1149 /* In case one didn't come up */
1150 if (!cpu_isset(cpu, cpu_callin_map)) {
1151 local_irq_enable();
1152 return -EIO;
1155 local_irq_enable();
1156 /* Unleash the CPU! */
1157 cpu_set(cpu, smp_commenced_mask);
1158 while (!cpu_isset(cpu, cpu_online_map))
1159 mb();
1160 return 0;
1163 void __init smp_cpus_done(unsigned int max_cpus)
1165 #ifdef CONFIG_X86_IO_APIC
1166 setup_ioapic_dest();
1167 #endif
1168 #ifdef CONFIG_X86_64
1169 zap_low_mappings();
1170 #endif
1171 /*
1172 * Disable executability of the SMP trampoline:
1173 */
1174 set_kernel_exec((unsigned long)trampoline_base, trampoline_exec);
1177 #if 0
1178 void __init smp_intr_init(void)
1180 /*
1181 * The reschedule interrupt is a CPU-to-CPU reschedule-helper
1182 * IPI, driven by wakeup.
1183 */
1184 set_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
1186 /* IPI for invalidation */
1187 set_intr_gate(INVALIDATE_TLB_VECTOR, invalidate_interrupt);
1189 /* IPI for generic function call */
1190 set_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
1192 #endif