ia64/linux-2.6.18-xen.hg

view arch/mips/kernel/smtc.c @ 452:c7ed6fe5dca0

kexec: dont initialise regions in reserve_memory()

There is no need to initialise efi_memmap_res and boot_param_res in
reserve_memory() for the initial xen domain as it is done in
machine_kexec_setup_resources() using values from the kexec hypercall.

Signed-off-by: Simon Horman <horms@verge.net.au>
author Keir Fraser <keir.fraser@citrix.com>
date Thu Feb 28 10:55:18 2008 +0000 (2008-02-28)
parents 831230e53067
children
line source
1 /* Copyright (C) 2004 Mips Technologies, Inc */
3 #include <linux/kernel.h>
4 #include <linux/sched.h>
5 #include <linux/cpumask.h>
6 #include <linux/interrupt.h>
8 #include <asm/cpu.h>
9 #include <asm/processor.h>
10 #include <asm/atomic.h>
11 #include <asm/system.h>
12 #include <asm/hardirq.h>
13 #include <asm/hazards.h>
14 #include <asm/mmu_context.h>
15 #include <asm/smp.h>
16 #include <asm/mipsregs.h>
17 #include <asm/cacheflush.h>
18 #include <asm/time.h>
19 #include <asm/addrspace.h>
20 #include <asm/smtc.h>
21 #include <asm/smtc_ipi.h>
22 #include <asm/smtc_proc.h>
24 /*
25 * This file should be built into the kernel only if CONFIG_MIPS_MT_SMTC is set.
26 */
28 /*
29 * MIPSCPU_INT_BASE is identically defined in both
30 * asm-mips/mips-boards/maltaint.h and asm-mips/mips-boards/simint.h,
31 * but as yet there's no properly organized include structure that
32 * will ensure that the right *int.h file will be included for a
33 * given platform build.
34 */
36 #define MIPSCPU_INT_BASE 16
38 #define MIPS_CPU_IPI_IRQ 1
40 #define LOCK_MT_PRA() \
41 local_irq_save(flags); \
42 mtflags = dmt()
44 #define UNLOCK_MT_PRA() \
45 emt(mtflags); \
46 local_irq_restore(flags)
48 #define LOCK_CORE_PRA() \
49 local_irq_save(flags); \
50 mtflags = dvpe()
52 #define UNLOCK_CORE_PRA() \
53 evpe(mtflags); \
54 local_irq_restore(flags)
56 /*
57 * Data structures purely associated with SMTC parallelism
58 */
61 /*
62 * Table for tracking ASIDs whose lifetime is prolonged.
63 */
65 asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS];
67 /*
68 * Clock interrupt "latch" buffers, per "CPU"
69 */
71 unsigned int ipi_timer_latch[NR_CPUS];
73 /*
74 * Number of InterProcessor Interupt (IPI) message buffers to allocate
75 */
77 #define IPIBUF_PER_CPU 4
79 struct smtc_ipi_q IPIQ[NR_CPUS];
80 struct smtc_ipi_q freeIPIq;
83 /* Forward declarations */
85 void ipi_decode(struct pt_regs *, struct smtc_ipi *);
86 void post_direct_ipi(int cpu, struct smtc_ipi *pipi);
87 void setup_cross_vpe_interrupts(void);
88 void init_smtc_stats(void);
90 /* Global SMTC Status */
92 unsigned int smtc_status = 0;
94 /* Boot command line configuration overrides */
96 static int vpelimit = 0;
97 static int tclimit = 0;
98 static int ipibuffers = 0;
99 static int nostlb = 0;
100 static int asidmask = 0;
101 unsigned long smtc_asid_mask = 0xff;
103 static int __init maxvpes(char *str)
104 {
105 get_option(&str, &vpelimit);
106 return 1;
107 }
109 static int __init maxtcs(char *str)
110 {
111 get_option(&str, &tclimit);
112 return 1;
113 }
115 static int __init ipibufs(char *str)
116 {
117 get_option(&str, &ipibuffers);
118 return 1;
119 }
121 static int __init stlb_disable(char *s)
122 {
123 nostlb = 1;
124 return 1;
125 }
127 static int __init asidmask_set(char *str)
128 {
129 get_option(&str, &asidmask);
130 switch (asidmask) {
131 case 0x1:
132 case 0x3:
133 case 0x7:
134 case 0xf:
135 case 0x1f:
136 case 0x3f:
137 case 0x7f:
138 case 0xff:
139 smtc_asid_mask = (unsigned long)asidmask;
140 break;
141 default:
142 printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask);
143 }
144 return 1;
145 }
147 __setup("maxvpes=", maxvpes);
148 __setup("maxtcs=", maxtcs);
149 __setup("ipibufs=", ipibufs);
150 __setup("nostlb", stlb_disable);
151 __setup("asidmask=", asidmask_set);
153 /* Enable additional debug checks before going into CPU idle loop */
154 #define SMTC_IDLE_HOOK_DEBUG
156 #ifdef SMTC_IDLE_HOOK_DEBUG
158 static int hang_trig = 0;
160 static int __init hangtrig_enable(char *s)
161 {
162 hang_trig = 1;
163 return 1;
164 }
167 __setup("hangtrig", hangtrig_enable);
169 #define DEFAULT_BLOCKED_IPI_LIMIT 32
171 static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT;
173 static int __init tintq(char *str)
174 {
175 get_option(&str, &timerq_limit);
176 return 1;
177 }
179 __setup("tintq=", tintq);
181 int imstuckcount[2][8];
182 /* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */
183 int vpemask[2][8] = {{0,1,1,0,0,0,0,1},{0,1,0,0,0,0,0,1}};
184 int tcnoprog[NR_CPUS];
185 static atomic_t idle_hook_initialized = {0};
186 static int clock_hang_reported[NR_CPUS];
188 #endif /* SMTC_IDLE_HOOK_DEBUG */
190 /* Initialize shared TLB - the should probably migrate to smtc_setup_cpus() */
192 void __init sanitize_tlb_entries(void)
193 {
194 printk("Deprecated sanitize_tlb_entries() invoked\n");
195 }
198 /*
199 * Configure shared TLB - VPC configuration bit must be set by caller
200 */
202 void smtc_configure_tlb(void)
203 {
204 int i,tlbsiz,vpes;
205 unsigned long mvpconf0;
206 unsigned long config1val;
208 /* Set up ASID preservation table */
209 for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) {
210 for(i = 0; i < MAX_SMTC_ASIDS; i++) {
211 smtc_live_asid[vpes][i] = 0;
212 }
213 }
214 mvpconf0 = read_c0_mvpconf0();
216 if ((vpes = ((mvpconf0 & MVPCONF0_PVPE)
217 >> MVPCONF0_PVPE_SHIFT) + 1) > 1) {
218 /* If we have multiple VPEs, try to share the TLB */
219 if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) {
220 /*
221 * If TLB sizing is programmable, shared TLB
222 * size is the total available complement.
223 * Otherwise, we have to take the sum of all
224 * static VPE TLB entries.
225 */
226 if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE)
227 >> MVPCONF0_PTLBE_SHIFT)) == 0) {
228 /*
229 * If there's more than one VPE, there had better
230 * be more than one TC, because we need one to bind
231 * to each VPE in turn to be able to read
232 * its configuration state!
233 */
234 settc(1);
235 /* Stop the TC from doing anything foolish */
236 write_tc_c0_tchalt(TCHALT_H);
237 mips_ihb();
238 /* No need to un-Halt - that happens later anyway */
239 for (i=0; i < vpes; i++) {
240 write_tc_c0_tcbind(i);
241 /*
242 * To be 100% sure we're really getting the right
243 * information, we exit the configuration state
244 * and do an IHB after each rebinding.
245 */
246 write_c0_mvpcontrol(
247 read_c0_mvpcontrol() & ~ MVPCONTROL_VPC );
248 mips_ihb();
249 /*
250 * Only count if the MMU Type indicated is TLB
251 */
252 if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) {
253 config1val = read_vpe_c0_config1();
254 tlbsiz += ((config1val >> 25) & 0x3f) + 1;
255 }
257 /* Put core back in configuration state */
258 write_c0_mvpcontrol(
259 read_c0_mvpcontrol() | MVPCONTROL_VPC );
260 mips_ihb();
261 }
262 }
263 write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB);
265 /*
266 * Setup kernel data structures to use software total,
267 * rather than read the per-VPE Config1 value. The values
268 * for "CPU 0" gets copied to all the other CPUs as part
269 * of their initialization in smtc_cpu_setup().
270 */
272 tlbsiz = tlbsiz & 0x3f; /* MIPS32 limits TLB indices to 64 */
273 cpu_data[0].tlbsize = tlbsiz;
274 smtc_status |= SMTC_TLB_SHARED;
276 printk("TLB of %d entry pairs shared by %d VPEs\n",
277 tlbsiz, vpes);
278 } else {
279 printk("WARNING: TLB Not Sharable on SMTC Boot!\n");
280 }
281 }
282 }
285 /*
286 * Incrementally build the CPU map out of constituent MIPS MT cores,
287 * using the specified available VPEs and TCs. Plaform code needs
288 * to ensure that each MIPS MT core invokes this routine on reset,
289 * one at a time(!).
290 *
291 * This version of the build_cpu_map and prepare_cpus routines assumes
292 * that *all* TCs of a MIPS MT core will be used for Linux, and that
293 * they will be spread across *all* available VPEs (to minimise the
294 * loss of efficiency due to exception service serialization).
295 * An improved version would pick up configuration information and
296 * possibly leave some TCs/VPEs as "slave" processors.
297 *
298 * Use c0_MVPConf0 to find out how many TCs are available, setting up
299 * phys_cpu_present_map and the logical/physical mappings.
300 */
302 int __init mipsmt_build_cpu_map(int start_cpu_slot)
303 {
304 int i, ntcs;
306 /*
307 * The CPU map isn't actually used for anything at this point,
308 * so it's not clear what else we should do apart from set
309 * everything up so that "logical" = "physical".
310 */
311 ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
312 for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) {
313 cpu_set(i, phys_cpu_present_map);
314 __cpu_number_map[i] = i;
315 __cpu_logical_map[i] = i;
316 }
317 /* Initialize map of CPUs with FPUs */
318 cpus_clear(mt_fpu_cpumask);
320 /* One of those TC's is the one booting, and not a secondary... */
321 printk("%i available secondary CPU TC(s)\n", i - 1);
323 return i;
324 }
326 /*
327 * Common setup before any secondaries are started
328 * Make sure all CPU's are in a sensible state before we boot any of the
329 * secondaries.
330 *
331 * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly
332 * as possible across the available VPEs.
333 */
335 static void smtc_tc_setup(int vpe, int tc, int cpu)
336 {
337 settc(tc);
338 write_tc_c0_tchalt(TCHALT_H);
339 mips_ihb();
340 write_tc_c0_tcstatus((read_tc_c0_tcstatus()
341 & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT))
342 | TCSTATUS_A);
343 write_tc_c0_tccontext(0);
344 /* Bind tc to vpe */
345 write_tc_c0_tcbind(vpe);
346 /* In general, all TCs should have the same cpu_data indications */
347 memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips));
348 /* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */
349 if (cpu_data[0].cputype == CPU_34K)
350 cpu_data[cpu].options &= ~MIPS_CPU_FPU;
351 cpu_data[cpu].vpe_id = vpe;
352 cpu_data[cpu].tc_id = tc;
353 }
356 void mipsmt_prepare_cpus(void)
357 {
358 int i, vpe, tc, ntc, nvpe, tcpervpe, slop, cpu;
359 unsigned long flags;
360 unsigned long val;
361 int nipi;
362 struct smtc_ipi *pipi;
364 /* disable interrupts so we can disable MT */
365 local_irq_save(flags);
366 /* disable MT so we can configure */
367 dvpe();
368 dmt();
370 spin_lock_init(&freeIPIq.lock);
372 /*
373 * We probably don't have as many VPEs as we do SMP "CPUs",
374 * but it's possible - and in any case we'll never use more!
375 */
376 for (i=0; i<NR_CPUS; i++) {
377 IPIQ[i].head = IPIQ[i].tail = NULL;
378 spin_lock_init(&IPIQ[i].lock);
379 IPIQ[i].depth = 0;
380 ipi_timer_latch[i] = 0;
381 }
383 /* cpu_data index starts at zero */
384 cpu = 0;
385 cpu_data[cpu].vpe_id = 0;
386 cpu_data[cpu].tc_id = 0;
387 cpu++;
389 /* Report on boot-time options */
390 mips_mt_set_cpuoptions ();
391 if (vpelimit > 0)
392 printk("Limit of %d VPEs set\n", vpelimit);
393 if (tclimit > 0)
394 printk("Limit of %d TCs set\n", tclimit);
395 if (nostlb) {
396 printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n");
397 }
398 if (asidmask)
399 printk("ASID mask value override to 0x%x\n", asidmask);
401 /* Temporary */
402 #ifdef SMTC_IDLE_HOOK_DEBUG
403 if (hang_trig)
404 printk("Logic Analyser Trigger on suspected TC hang\n");
405 #endif /* SMTC_IDLE_HOOK_DEBUG */
407 /* Put MVPE's into 'configuration state' */
408 write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC );
410 val = read_c0_mvpconf0();
411 nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1;
412 if (vpelimit > 0 && nvpe > vpelimit)
413 nvpe = vpelimit;
414 ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
415 if (ntc > NR_CPUS)
416 ntc = NR_CPUS;
417 if (tclimit > 0 && ntc > tclimit)
418 ntc = tclimit;
419 tcpervpe = ntc / nvpe;
420 slop = ntc % nvpe; /* Residual TCs, < NVPE */
422 /* Set up shared TLB */
423 smtc_configure_tlb();
425 for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) {
426 /*
427 * Set the MVP bits.
428 */
429 settc(tc);
430 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_MVP);
431 if (vpe != 0)
432 printk(", ");
433 printk("VPE %d: TC", vpe);
434 for (i = 0; i < tcpervpe; i++) {
435 /*
436 * TC 0 is bound to VPE 0 at reset,
437 * and is presumably executing this
438 * code. Leave it alone!
439 */
440 if (tc != 0) {
441 smtc_tc_setup(vpe,tc, cpu);
442 cpu++;
443 }
444 printk(" %d", tc);
445 tc++;
446 }
447 if (slop) {
448 if (tc != 0) {
449 smtc_tc_setup(vpe,tc, cpu);
450 cpu++;
451 }
452 printk(" %d", tc);
453 tc++;
454 slop--;
455 }
456 if (vpe != 0) {
457 /*
458 * Clear any stale software interrupts from VPE's Cause
459 */
460 write_vpe_c0_cause(0);
462 /*
463 * Clear ERL/EXL of VPEs other than 0
464 * and set restricted interrupt enable/mask.
465 */
466 write_vpe_c0_status((read_vpe_c0_status()
467 & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM))
468 | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7
469 | ST0_IE));
470 /*
471 * set config to be the same as vpe0,
472 * particularly kseg0 coherency alg
473 */
474 write_vpe_c0_config(read_c0_config());
475 /* Clear any pending timer interrupt */
476 write_vpe_c0_compare(0);
477 /* Propagate Config7 */
478 write_vpe_c0_config7(read_c0_config7());
479 }
480 /* enable multi-threading within VPE */
481 write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE);
482 /* enable the VPE */
483 write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA);
484 }
486 /*
487 * Pull any physically present but unused TCs out of circulation.
488 */
489 while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) {
490 cpu_clear(tc, phys_cpu_present_map);
491 cpu_clear(tc, cpu_present_map);
492 tc++;
493 }
495 /* release config state */
496 write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC );
498 printk("\n");
500 /* Set up coprocessor affinity CPU mask(s) */
502 for (tc = 0; tc < ntc; tc++) {
503 if (cpu_data[tc].options & MIPS_CPU_FPU)
504 cpu_set(tc, mt_fpu_cpumask);
505 }
507 /* set up ipi interrupts... */
509 /* If we have multiple VPEs running, set up the cross-VPE interrupt */
511 if (nvpe > 1)
512 setup_cross_vpe_interrupts();
514 /* Set up queue of free IPI "messages". */
515 nipi = NR_CPUS * IPIBUF_PER_CPU;
516 if (ipibuffers > 0)
517 nipi = ipibuffers;
519 pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL);
520 if (pipi == NULL)
521 panic("kmalloc of IPI message buffers failed\n");
522 else
523 printk("IPI buffer pool of %d buffers\n", nipi);
524 for (i = 0; i < nipi; i++) {
525 smtc_ipi_nq(&freeIPIq, pipi);
526 pipi++;
527 }
529 /* Arm multithreading and enable other VPEs - but all TCs are Halted */
530 emt(EMT_ENABLE);
531 evpe(EVPE_ENABLE);
532 local_irq_restore(flags);
533 /* Initialize SMTC /proc statistics/diagnostics */
534 init_smtc_stats();
535 }
538 /*
539 * Setup the PC, SP, and GP of a secondary processor and start it
540 * running!
541 * smp_bootstrap is the place to resume from
542 * __KSTK_TOS(idle) is apparently the stack pointer
543 * (unsigned long)idle->thread_info the gp
544 *
545 */
546 void smtc_boot_secondary(int cpu, struct task_struct *idle)
547 {
548 extern u32 kernelsp[NR_CPUS];
549 long flags;
550 int mtflags;
552 LOCK_MT_PRA();
553 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
554 dvpe();
555 }
556 settc(cpu_data[cpu].tc_id);
558 /* pc */
559 write_tc_c0_tcrestart((unsigned long)&smp_bootstrap);
561 /* stack pointer */
562 kernelsp[cpu] = __KSTK_TOS(idle);
563 write_tc_gpr_sp(__KSTK_TOS(idle));
565 /* global pointer */
566 write_tc_gpr_gp((unsigned long)idle->thread_info);
568 smtc_status |= SMTC_MTC_ACTIVE;
569 write_tc_c0_tchalt(0);
570 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
571 evpe(EVPE_ENABLE);
572 }
573 UNLOCK_MT_PRA();
574 }
576 void smtc_init_secondary(void)
577 {
578 /*
579 * Start timer on secondary VPEs if necessary.
580 * plat_timer_setup has already have been invoked by init/main
581 * on "boot" TC. Like per_cpu_trap_init() hack, this assumes that
582 * SMTC init code assigns TCs consdecutively and in ascending order
583 * to across available VPEs.
584 */
585 if (((read_c0_tcbind() & TCBIND_CURTC) != 0) &&
586 ((read_c0_tcbind() & TCBIND_CURVPE)
587 != cpu_data[smp_processor_id() - 1].vpe_id)){
588 write_c0_compare (read_c0_count() + mips_hpt_frequency/HZ);
589 }
591 local_irq_enable();
592 }
594 void smtc_smp_finish(void)
595 {
596 printk("TC %d going on-line as CPU %d\n",
597 cpu_data[smp_processor_id()].tc_id, smp_processor_id());
598 }
600 void smtc_cpus_done(void)
601 {
602 }
604 /*
605 * Support for SMTC-optimized driver IRQ registration
606 */
608 /*
609 * SMTC Kernel needs to manipulate low-level CPU interrupt mask
610 * in do_IRQ. These are passed in setup_irq_smtc() and stored
611 * in this table.
612 */
614 int setup_irq_smtc(unsigned int irq, struct irqaction * new,
615 unsigned long hwmask)
616 {
617 irq_hwmask[irq] = hwmask;
619 return setup_irq(irq, new);
620 }
622 /*
623 * IPI model for SMTC is tricky, because interrupts aren't TC-specific.
624 * Within a VPE one TC can interrupt another by different approaches.
625 * The easiest to get right would probably be to make all TCs except
626 * the target IXMT and set a software interrupt, but an IXMT-based
627 * scheme requires that a handler must run before a new IPI could
628 * be sent, which would break the "broadcast" loops in MIPS MT.
629 * A more gonzo approach within a VPE is to halt the TC, extract
630 * its Restart, Status, and a couple of GPRs, and program the Restart
631 * address to emulate an interrupt.
632 *
633 * Within a VPE, one can be confident that the target TC isn't in
634 * a critical EXL state when halted, since the write to the Halt
635 * register could not have issued on the writing thread if the
636 * halting thread had EXL set. So k0 and k1 of the target TC
637 * can be used by the injection code. Across VPEs, one can't
638 * be certain that the target TC isn't in a critical exception
639 * state. So we try a two-step process of sending a software
640 * interrupt to the target VPE, which either handles the event
641 * itself (if it was the target) or injects the event within
642 * the VPE.
643 */
645 void smtc_ipi_qdump(void)
646 {
647 int i;
649 for (i = 0; i < NR_CPUS ;i++) {
650 printk("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n",
651 i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail,
652 IPIQ[i].depth);
653 }
654 }
656 /*
657 * The standard atomic.h primitives don't quite do what we want
658 * here: We need an atomic add-and-return-previous-value (which
659 * could be done with atomic_add_return and a decrement) and an
660 * atomic set/zero-and-return-previous-value (which can't really
661 * be done with the atomic.h primitives). And since this is
662 * MIPS MT, we can assume that we have LL/SC.
663 */
664 static __inline__ int atomic_postincrement(unsigned int *pv)
665 {
666 unsigned long result;
668 unsigned long temp;
670 __asm__ __volatile__(
671 "1: ll %0, %2 \n"
672 " addu %1, %0, 1 \n"
673 " sc %1, %2 \n"
674 " beqz %1, 1b \n"
675 " sync \n"
676 : "=&r" (result), "=&r" (temp), "=m" (*pv)
677 : "m" (*pv)
678 : "memory");
680 return result;
681 }
683 /* No longer used in IPI dispatch, but retained for future recycling */
685 static __inline__ int atomic_postclear(unsigned int *pv)
686 {
687 unsigned long result;
689 unsigned long temp;
691 __asm__ __volatile__(
692 "1: ll %0, %2 \n"
693 " or %1, $0, $0 \n"
694 " sc %1, %2 \n"
695 " beqz %1, 1b \n"
696 " sync \n"
697 : "=&r" (result), "=&r" (temp), "=m" (*pv)
698 : "m" (*pv)
699 : "memory");
701 return result;
702 }
705 void smtc_send_ipi(int cpu, int type, unsigned int action)
706 {
707 int tcstatus;
708 struct smtc_ipi *pipi;
709 long flags;
710 int mtflags;
712 if (cpu == smp_processor_id()) {
713 printk("Cannot Send IPI to self!\n");
714 return;
715 }
716 /* Set up a descriptor, to be delivered either promptly or queued */
717 pipi = smtc_ipi_dq(&freeIPIq);
718 if (pipi == NULL) {
719 bust_spinlocks(1);
720 mips_mt_regdump(dvpe());
721 panic("IPI Msg. Buffers Depleted\n");
722 }
723 pipi->type = type;
724 pipi->arg = (void *)action;
725 pipi->dest = cpu;
726 if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
727 /* If not on same VPE, enqueue and send cross-VPE interupt */
728 smtc_ipi_nq(&IPIQ[cpu], pipi);
729 LOCK_CORE_PRA();
730 settc(cpu_data[cpu].tc_id);
731 write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1);
732 UNLOCK_CORE_PRA();
733 } else {
734 /*
735 * Not sufficient to do a LOCK_MT_PRA (dmt) here,
736 * since ASID shootdown on the other VPE may
737 * collide with this operation.
738 */
739 LOCK_CORE_PRA();
740 settc(cpu_data[cpu].tc_id);
741 /* Halt the targeted TC */
742 write_tc_c0_tchalt(TCHALT_H);
743 mips_ihb();
745 /*
746 * Inspect TCStatus - if IXMT is set, we have to queue
747 * a message. Otherwise, we set up the "interrupt"
748 * of the other TC
749 */
750 tcstatus = read_tc_c0_tcstatus();
752 if ((tcstatus & TCSTATUS_IXMT) != 0) {
753 /*
754 * Spin-waiting here can deadlock,
755 * so we queue the message for the target TC.
756 */
757 write_tc_c0_tchalt(0);
758 UNLOCK_CORE_PRA();
759 /* Try to reduce redundant timer interrupt messages */
760 if (type == SMTC_CLOCK_TICK) {
761 if (atomic_postincrement(&ipi_timer_latch[cpu])!=0){
762 smtc_ipi_nq(&freeIPIq, pipi);
763 return;
764 }
765 }
766 smtc_ipi_nq(&IPIQ[cpu], pipi);
767 } else {
768 post_direct_ipi(cpu, pipi);
769 write_tc_c0_tchalt(0);
770 UNLOCK_CORE_PRA();
771 }
772 }
773 }
775 /*
776 * Send IPI message to Halted TC, TargTC/TargVPE already having been set
777 */
778 void post_direct_ipi(int cpu, struct smtc_ipi *pipi)
779 {
780 struct pt_regs *kstack;
781 unsigned long tcstatus;
782 unsigned long tcrestart;
783 extern u32 kernelsp[NR_CPUS];
784 extern void __smtc_ipi_vector(void);
786 /* Extract Status, EPC from halted TC */
787 tcstatus = read_tc_c0_tcstatus();
788 tcrestart = read_tc_c0_tcrestart();
789 /* If TCRestart indicates a WAIT instruction, advance the PC */
790 if ((tcrestart & 0x80000000)
791 && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) {
792 tcrestart += 4;
793 }
794 /*
795 * Save on TC's future kernel stack
796 *
797 * CU bit of Status is indicator that TC was
798 * already running on a kernel stack...
799 */
800 if (tcstatus & ST0_CU0) {
801 /* Note that this "- 1" is pointer arithmetic */
802 kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1;
803 } else {
804 kstack = ((struct pt_regs *)kernelsp[cpu]) - 1;
805 }
807 kstack->cp0_epc = (long)tcrestart;
808 /* Save TCStatus */
809 kstack->cp0_tcstatus = tcstatus;
810 /* Pass token of operation to be performed kernel stack pad area */
811 kstack->pad0[4] = (unsigned long)pipi;
812 /* Pass address of function to be called likewise */
813 kstack->pad0[5] = (unsigned long)&ipi_decode;
814 /* Set interrupt exempt and kernel mode */
815 tcstatus |= TCSTATUS_IXMT;
816 tcstatus &= ~TCSTATUS_TKSU;
817 write_tc_c0_tcstatus(tcstatus);
818 ehb();
819 /* Set TC Restart address to be SMTC IPI vector */
820 write_tc_c0_tcrestart(__smtc_ipi_vector);
821 }
823 void ipi_resched_interrupt(struct pt_regs *regs)
824 {
825 /* Return from interrupt should be enough to cause scheduler check */
826 }
829 void ipi_call_interrupt(struct pt_regs *regs)
830 {
831 /* Invoke generic function invocation code in smp.c */
832 smp_call_function_interrupt();
833 }
835 void ipi_decode(struct pt_regs *regs, struct smtc_ipi *pipi)
836 {
837 void *arg_copy = pipi->arg;
838 int type_copy = pipi->type;
839 int dest_copy = pipi->dest;
841 smtc_ipi_nq(&freeIPIq, pipi);
842 switch (type_copy) {
843 case SMTC_CLOCK_TICK:
844 /* Invoke Clock "Interrupt" */
845 ipi_timer_latch[dest_copy] = 0;
846 #ifdef SMTC_IDLE_HOOK_DEBUG
847 clock_hang_reported[dest_copy] = 0;
848 #endif /* SMTC_IDLE_HOOK_DEBUG */
849 local_timer_interrupt(0, NULL, regs);
850 break;
851 case LINUX_SMP_IPI:
852 switch ((int)arg_copy) {
853 case SMP_RESCHEDULE_YOURSELF:
854 ipi_resched_interrupt(regs);
855 break;
856 case SMP_CALL_FUNCTION:
857 ipi_call_interrupt(regs);
858 break;
859 default:
860 printk("Impossible SMTC IPI Argument 0x%x\n",
861 (int)arg_copy);
862 break;
863 }
864 break;
865 default:
866 printk("Impossible SMTC IPI Type 0x%x\n", type_copy);
867 break;
868 }
869 }
871 void deferred_smtc_ipi(struct pt_regs *regs)
872 {
873 struct smtc_ipi *pipi;
874 unsigned long flags;
875 /* DEBUG */
876 int q = smp_processor_id();
878 /*
879 * Test is not atomic, but much faster than a dequeue,
880 * and the vast majority of invocations will have a null queue.
881 */
882 if (IPIQ[q].head != NULL) {
883 while((pipi = smtc_ipi_dq(&IPIQ[q])) != NULL) {
884 /* ipi_decode() should be called with interrupts off */
885 local_irq_save(flags);
886 ipi_decode(regs, pipi);
887 local_irq_restore(flags);
888 }
889 }
890 }
892 /*
893 * Send clock tick to all TCs except the one executing the funtion
894 */
896 void smtc_timer_broadcast(int vpe)
897 {
898 int cpu;
899 int myTC = cpu_data[smp_processor_id()].tc_id;
900 int myVPE = cpu_data[smp_processor_id()].vpe_id;
902 smtc_cpu_stats[smp_processor_id()].timerints++;
904 for_each_online_cpu(cpu) {
905 if (cpu_data[cpu].vpe_id == myVPE &&
906 cpu_data[cpu].tc_id != myTC)
907 smtc_send_ipi(cpu, SMTC_CLOCK_TICK, 0);
908 }
909 }
911 /*
912 * Cross-VPE interrupts in the SMTC prototype use "software interrupts"
913 * set via cross-VPE MTTR manipulation of the Cause register. It would be
914 * in some regards preferable to have external logic for "doorbell" hardware
915 * interrupts.
916 */
918 static int cpu_ipi_irq = MIPSCPU_INT_BASE + MIPS_CPU_IPI_IRQ;
920 static irqreturn_t ipi_interrupt(int irq, void *dev_idm, struct pt_regs *regs)
921 {
922 int my_vpe = cpu_data[smp_processor_id()].vpe_id;
923 int my_tc = cpu_data[smp_processor_id()].tc_id;
924 int cpu;
925 struct smtc_ipi *pipi;
926 unsigned long tcstatus;
927 int sent;
928 long flags;
929 unsigned int mtflags;
930 unsigned int vpflags;
932 /*
933 * So long as cross-VPE interrupts are done via
934 * MFTR/MTTR read-modify-writes of Cause, we need
935 * to stop other VPEs whenever the local VPE does
936 * anything similar.
937 */
938 local_irq_save(flags);
939 vpflags = dvpe();
940 clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ);
941 set_c0_status(0x100 << MIPS_CPU_IPI_IRQ);
942 irq_enable_hazard();
943 evpe(vpflags);
944 local_irq_restore(flags);
946 /*
947 * Cross-VPE Interrupt handler: Try to directly deliver IPIs
948 * queued for TCs on this VPE other than the current one.
949 * Return-from-interrupt should cause us to drain the queue
950 * for the current TC, so we ought not to have to do it explicitly here.
951 */
953 for_each_online_cpu(cpu) {
954 if (cpu_data[cpu].vpe_id != my_vpe)
955 continue;
957 pipi = smtc_ipi_dq(&IPIQ[cpu]);
958 if (pipi != NULL) {
959 if (cpu_data[cpu].tc_id != my_tc) {
960 sent = 0;
961 LOCK_MT_PRA();
962 settc(cpu_data[cpu].tc_id);
963 write_tc_c0_tchalt(TCHALT_H);
964 mips_ihb();
965 tcstatus = read_tc_c0_tcstatus();
966 if ((tcstatus & TCSTATUS_IXMT) == 0) {
967 post_direct_ipi(cpu, pipi);
968 sent = 1;
969 }
970 write_tc_c0_tchalt(0);
971 UNLOCK_MT_PRA();
972 if (!sent) {
973 smtc_ipi_req(&IPIQ[cpu], pipi);
974 }
975 } else {
976 /*
977 * ipi_decode() should be called
978 * with interrupts off
979 */
980 local_irq_save(flags);
981 ipi_decode(regs, pipi);
982 local_irq_restore(flags);
983 }
984 }
985 }
987 return IRQ_HANDLED;
988 }
990 static void ipi_irq_dispatch(struct pt_regs *regs)
991 {
992 do_IRQ(cpu_ipi_irq, regs);
993 }
995 static struct irqaction irq_ipi;
997 void setup_cross_vpe_interrupts(void)
998 {
999 if (!cpu_has_vint)
1000 panic("SMTC Kernel requires Vectored Interupt support");
1002 set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch);
1004 irq_ipi.handler = ipi_interrupt;
1005 irq_ipi.flags = IRQF_DISABLED;
1006 irq_ipi.name = "SMTC_IPI";
1008 setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ));
1010 irq_desc[cpu_ipi_irq].status |= IRQ_PER_CPU;
1013 /*
1014 * SMTC-specific hacks invoked from elsewhere in the kernel.
1015 */
1017 void smtc_idle_loop_hook(void)
1019 #ifdef SMTC_IDLE_HOOK_DEBUG
1020 int im;
1021 int flags;
1022 int mtflags;
1023 int bit;
1024 int vpe;
1025 int tc;
1026 int hook_ntcs;
1027 /*
1028 * printk within DMT-protected regions can deadlock,
1029 * so buffer diagnostic messages for later output.
1030 */
1031 char *pdb_msg;
1032 char id_ho_db_msg[768]; /* worst-case use should be less than 700 */
1034 if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */
1035 if (atomic_add_return(1, &idle_hook_initialized) == 1) {
1036 int mvpconf0;
1037 /* Tedious stuff to just do once */
1038 mvpconf0 = read_c0_mvpconf0();
1039 hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
1040 if (hook_ntcs > NR_CPUS)
1041 hook_ntcs = NR_CPUS;
1042 for (tc = 0; tc < hook_ntcs; tc++) {
1043 tcnoprog[tc] = 0;
1044 clock_hang_reported[tc] = 0;
1046 for (vpe = 0; vpe < 2; vpe++)
1047 for (im = 0; im < 8; im++)
1048 imstuckcount[vpe][im] = 0;
1049 printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs);
1050 atomic_set(&idle_hook_initialized, 1000);
1051 } else {
1052 /* Someone else is initializing in parallel - let 'em finish */
1053 while (atomic_read(&idle_hook_initialized) < 1000)
1058 /* Have we stupidly left IXMT set somewhere? */
1059 if (read_c0_tcstatus() & 0x400) {
1060 write_c0_tcstatus(read_c0_tcstatus() & ~0x400);
1061 ehb();
1062 printk("Dangling IXMT in cpu_idle()\n");
1065 /* Have we stupidly left an IM bit turned off? */
1066 #define IM_LIMIT 2000
1067 local_irq_save(flags);
1068 mtflags = dmt();
1069 pdb_msg = &id_ho_db_msg[0];
1070 im = read_c0_status();
1071 vpe = cpu_data[smp_processor_id()].vpe_id;
1072 for (bit = 0; bit < 8; bit++) {
1073 /*
1074 * In current prototype, I/O interrupts
1075 * are masked for VPE > 0
1076 */
1077 if (vpemask[vpe][bit]) {
1078 if (!(im & (0x100 << bit)))
1079 imstuckcount[vpe][bit]++;
1080 else
1081 imstuckcount[vpe][bit] = 0;
1082 if (imstuckcount[vpe][bit] > IM_LIMIT) {
1083 set_c0_status(0x100 << bit);
1084 ehb();
1085 imstuckcount[vpe][bit] = 0;
1086 pdb_msg += sprintf(pdb_msg,
1087 "Dangling IM %d fixed for VPE %d\n", bit,
1088 vpe);
1093 /*
1094 * Now that we limit outstanding timer IPIs, check for hung TC
1095 */
1096 for (tc = 0; tc < NR_CPUS; tc++) {
1097 /* Don't check ourself - we'll dequeue IPIs just below */
1098 if ((tc != smp_processor_id()) &&
1099 ipi_timer_latch[tc] > timerq_limit) {
1100 if (clock_hang_reported[tc] == 0) {
1101 pdb_msg += sprintf(pdb_msg,
1102 "TC %d looks hung with timer latch at %d\n",
1103 tc, ipi_timer_latch[tc]);
1104 clock_hang_reported[tc]++;
1108 emt(mtflags);
1109 local_irq_restore(flags);
1110 if (pdb_msg != &id_ho_db_msg[0])
1111 printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg);
1112 #endif /* SMTC_IDLE_HOOK_DEBUG */
1113 /*
1114 * To the extent that we've ever turned interrupts off,
1115 * we may have accumulated deferred IPIs. This is subtle.
1116 * If we use the smtc_ipi_qdepth() macro, we'll get an
1117 * exact number - but we'll also disable interrupts
1118 * and create a window of failure where a new IPI gets
1119 * queued after we test the depth but before we re-enable
1120 * interrupts. So long as IXMT never gets set, however,
1121 * we should be OK: If we pick up something and dispatch
1122 * it here, that's great. If we see nothing, but concurrent
1123 * with this operation, another TC sends us an IPI, IXMT
1124 * is clear, and we'll handle it as a real pseudo-interrupt
1125 * and not a pseudo-pseudo interrupt.
1126 */
1127 if (IPIQ[smp_processor_id()].depth > 0) {
1128 struct smtc_ipi *pipi;
1129 extern void self_ipi(struct smtc_ipi *);
1131 if ((pipi = smtc_ipi_dq(&IPIQ[smp_processor_id()])) != NULL) {
1132 self_ipi(pipi);
1133 smtc_cpu_stats[smp_processor_id()].selfipis++;
1138 void smtc_soft_dump(void)
1140 int i;
1142 printk("Counter Interrupts taken per CPU (TC)\n");
1143 for (i=0; i < NR_CPUS; i++) {
1144 printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints);
1146 printk("Self-IPI invocations:\n");
1147 for (i=0; i < NR_CPUS; i++) {
1148 printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis);
1150 smtc_ipi_qdump();
1151 printk("Timer IPI Backlogs:\n");
1152 for (i=0; i < NR_CPUS; i++) {
1153 printk("%d: %d\n", i, ipi_timer_latch[i]);
1155 printk("%d Recoveries of \"stolen\" FPU\n",
1156 atomic_read(&smtc_fpu_recoveries));
1160 /*
1161 * TLB management routines special to SMTC
1162 */
1164 void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu)
1166 unsigned long flags, mtflags, tcstat, prevhalt, asid;
1167 int tlb, i;
1169 /*
1170 * It would be nice to be able to use a spinlock here,
1171 * but this is invoked from within TLB flush routines
1172 * that protect themselves with DVPE, so if a lock is
1173 * held by another TC, it'll never be freed.
1175 * DVPE/DMT must not be done with interrupts enabled,
1176 * so even so most callers will already have disabled
1177 * them, let's be really careful...
1178 */
1180 local_irq_save(flags);
1181 if (smtc_status & SMTC_TLB_SHARED) {
1182 mtflags = dvpe();
1183 tlb = 0;
1184 } else {
1185 mtflags = dmt();
1186 tlb = cpu_data[cpu].vpe_id;
1188 asid = asid_cache(cpu);
1190 do {
1191 if (!((asid += ASID_INC) & ASID_MASK) ) {
1192 if (cpu_has_vtag_icache)
1193 flush_icache_all();
1194 /* Traverse all online CPUs (hack requires contigous range) */
1195 for (i = 0; i < num_online_cpus(); i++) {
1196 /*
1197 * We don't need to worry about our own CPU, nor those of
1198 * CPUs who don't share our TLB.
1199 */
1200 if ((i != smp_processor_id()) &&
1201 ((smtc_status & SMTC_TLB_SHARED) ||
1202 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) {
1203 settc(cpu_data[i].tc_id);
1204 prevhalt = read_tc_c0_tchalt() & TCHALT_H;
1205 if (!prevhalt) {
1206 write_tc_c0_tchalt(TCHALT_H);
1207 mips_ihb();
1209 tcstat = read_tc_c0_tcstatus();
1210 smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i);
1211 if (!prevhalt)
1212 write_tc_c0_tchalt(0);
1215 if (!asid) /* fix version if needed */
1216 asid = ASID_FIRST_VERSION;
1217 local_flush_tlb_all(); /* start new asid cycle */
1219 } while (smtc_live_asid[tlb][(asid & ASID_MASK)]);
1221 /*
1222 * SMTC shares the TLB within VPEs and possibly across all VPEs.
1223 */
1224 for (i = 0; i < num_online_cpus(); i++) {
1225 if ((smtc_status & SMTC_TLB_SHARED) ||
1226 (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))
1227 cpu_context(i, mm) = asid_cache(i) = asid;
1230 if (smtc_status & SMTC_TLB_SHARED)
1231 evpe(mtflags);
1232 else
1233 emt(mtflags);
1234 local_irq_restore(flags);
1237 /*
1238 * Invoked from macros defined in mmu_context.h
1239 * which must already have disabled interrupts
1240 * and done a DVPE or DMT as appropriate.
1241 */
1243 void smtc_flush_tlb_asid(unsigned long asid)
1245 int entry;
1246 unsigned long ehi;
1248 entry = read_c0_wired();
1250 /* Traverse all non-wired entries */
1251 while (entry < current_cpu_data.tlbsize) {
1252 write_c0_index(entry);
1253 ehb();
1254 tlb_read();
1255 ehb();
1256 ehi = read_c0_entryhi();
1257 if ((ehi & ASID_MASK) == asid) {
1258 /*
1259 * Invalidate only entries with specified ASID,
1260 * makiing sure all entries differ.
1261 */
1262 write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
1263 write_c0_entrylo0(0);
1264 write_c0_entrylo1(0);
1265 mtc0_tlbw_hazard();
1266 tlb_write_indexed();
1268 entry++;
1270 write_c0_index(PARKED_INDEX);
1271 tlbw_use_hazard();
1274 /*
1275 * Support for single-threading cache flush operations.
1276 */
1278 int halt_state_save[NR_CPUS];
1280 /*
1281 * To really, really be sure that nothing is being done
1282 * by other TCs, halt them all. This code assumes that
1283 * a DVPE has already been done, so while their Halted
1284 * state is theoretically architecturally unstable, in
1285 * practice, it's not going to change while we're looking
1286 * at it.
1287 */
1289 void smtc_cflush_lockdown(void)
1291 int cpu;
1293 for_each_online_cpu(cpu) {
1294 if (cpu != smp_processor_id()) {
1295 settc(cpu_data[cpu].tc_id);
1296 halt_state_save[cpu] = read_tc_c0_tchalt();
1297 write_tc_c0_tchalt(TCHALT_H);
1300 mips_ihb();
1303 /* It would be cheating to change the cpu_online states during a flush! */
1305 void smtc_cflush_release(void)
1307 int cpu;
1309 /*
1310 * Start with a hazard barrier to ensure
1311 * that all CACHE ops have played through.
1312 */
1313 mips_ihb();
1315 for_each_online_cpu(cpu) {
1316 if (cpu != smp_processor_id()) {
1317 settc(cpu_data[cpu].tc_id);
1318 write_tc_c0_tchalt(halt_state_save[cpu]);
1321 mips_ihb();