ia64/linux-2.6.18-xen.hg

view arch/sparc64/kernel/traps.c @ 897:329ea0ccb344

balloon: try harder to balloon up under memory pressure.

Currently if the balloon driver is unable to increase the guest's
reservation it assumes the failure was due to reaching its full
allocation, gives up on the ballooning operation and records the limit
it reached as the "hard limit". The driver will not try again until
the target is set again (even to the same value).

However it is possible that ballooning has in fact failed due to
memory pressure in the host and therefore it is desirable to keep
attempting to reach the target in case memory becomes available. The
most likely scenario is that some guests are ballooning down while
others are ballooning up and therefore there is temporary memory
pressure while things stabilise. You would not expect a well behaved
toolstack to ask a domain to balloon to more than its allocation nor
would you expect it to deliberately over-commit memory by setting
balloon targets which exceed the total host memory.

This patch drops the concept of a hard limit and causes the balloon
driver to retry increasing the reservation on a timer in the same
manner as when decreasing the reservation.

Also if we partially succeed in increasing the reservation
(i.e. receive less pages than we asked for) then we may as well keep
those pages rather than returning them to Xen.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri Jun 05 14:01:20 2009 +0100 (2009-06-05)
parents 831230e53067
children
line source
1 /* $Id: traps.c,v 1.85 2002/02/09 19:49:31 davem Exp $
2 * arch/sparc64/kernel/traps.c
3 *
4 * Copyright (C) 1995,1997 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com)
6 */
8 /*
9 * I like traps on v9, :))))
10 */
12 #include <linux/module.h>
13 #include <linux/sched.h> /* for jiffies */
14 #include <linux/kernel.h>
15 #include <linux/kallsyms.h>
16 #include <linux/signal.h>
17 #include <linux/smp.h>
18 #include <linux/smp_lock.h>
19 #include <linux/mm.h>
20 #include <linux/init.h>
22 #include <asm/delay.h>
23 #include <asm/system.h>
24 #include <asm/ptrace.h>
25 #include <asm/oplib.h>
26 #include <asm/page.h>
27 #include <asm/pgtable.h>
28 #include <asm/unistd.h>
29 #include <asm/uaccess.h>
30 #include <asm/fpumacro.h>
31 #include <asm/lsu.h>
32 #include <asm/dcu.h>
33 #include <asm/estate.h>
34 #include <asm/chafsr.h>
35 #include <asm/sfafsr.h>
36 #include <asm/psrcompat.h>
37 #include <asm/processor.h>
38 #include <asm/timer.h>
39 #include <asm/kdebug.h>
40 #include <asm/head.h>
41 #ifdef CONFIG_KMOD
42 #include <linux/kmod.h>
43 #endif
44 #include <asm/prom.h>
46 ATOMIC_NOTIFIER_HEAD(sparc64die_chain);
48 int register_die_notifier(struct notifier_block *nb)
49 {
50 return atomic_notifier_chain_register(&sparc64die_chain, nb);
51 }
52 EXPORT_SYMBOL(register_die_notifier);
54 int unregister_die_notifier(struct notifier_block *nb)
55 {
56 return atomic_notifier_chain_unregister(&sparc64die_chain, nb);
57 }
58 EXPORT_SYMBOL(unregister_die_notifier);
60 /* When an irrecoverable trap occurs at tl > 0, the trap entry
61 * code logs the trap state registers at every level in the trap
62 * stack. It is found at (pt_regs + sizeof(pt_regs)) and the layout
63 * is as follows:
64 */
65 struct tl1_traplog {
66 struct {
67 unsigned long tstate;
68 unsigned long tpc;
69 unsigned long tnpc;
70 unsigned long tt;
71 } trapstack[4];
72 unsigned long tl;
73 };
75 static void dump_tl1_traplog(struct tl1_traplog *p)
76 {
77 int i, limit;
79 printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, "
80 "dumping track stack.\n", p->tl);
82 limit = (tlb_type == hypervisor) ? 2 : 4;
83 for (i = 0; i < limit; i++) {
84 printk(KERN_EMERG
85 "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] "
86 "TNPC[%016lx] TT[%lx]\n",
87 i + 1,
88 p->trapstack[i].tstate, p->trapstack[i].tpc,
89 p->trapstack[i].tnpc, p->trapstack[i].tt);
90 }
91 }
93 void do_call_debug(struct pt_regs *regs)
94 {
95 notify_die(DIE_CALL, "debug call", regs, 0, 255, SIGINT);
96 }
98 void bad_trap(struct pt_regs *regs, long lvl)
99 {
100 char buffer[32];
101 siginfo_t info;
103 if (notify_die(DIE_TRAP, "bad trap", regs,
104 0, lvl, SIGTRAP) == NOTIFY_STOP)
105 return;
107 if (lvl < 0x100) {
108 sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl);
109 die_if_kernel(buffer, regs);
110 }
112 lvl -= 0x100;
113 if (regs->tstate & TSTATE_PRIV) {
114 sprintf(buffer, "Kernel bad sw trap %lx", lvl);
115 die_if_kernel(buffer, regs);
116 }
117 if (test_thread_flag(TIF_32BIT)) {
118 regs->tpc &= 0xffffffff;
119 regs->tnpc &= 0xffffffff;
120 }
121 info.si_signo = SIGILL;
122 info.si_errno = 0;
123 info.si_code = ILL_ILLTRP;
124 info.si_addr = (void __user *)regs->tpc;
125 info.si_trapno = lvl;
126 force_sig_info(SIGILL, &info, current);
127 }
129 void bad_trap_tl1(struct pt_regs *regs, long lvl)
130 {
131 char buffer[32];
133 if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs,
134 0, lvl, SIGTRAP) == NOTIFY_STOP)
135 return;
137 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
139 sprintf (buffer, "Bad trap %lx at tl>0", lvl);
140 die_if_kernel (buffer, regs);
141 }
143 #ifdef CONFIG_DEBUG_BUGVERBOSE
144 void do_BUG(const char *file, int line)
145 {
146 bust_spinlocks(1);
147 printk("kernel BUG at %s:%d!\n", file, line);
148 }
149 #endif
151 void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
152 {
153 siginfo_t info;
155 if (notify_die(DIE_TRAP, "instruction access exception", regs,
156 0, 0x8, SIGTRAP) == NOTIFY_STOP)
157 return;
159 if (regs->tstate & TSTATE_PRIV) {
160 printk("spitfire_insn_access_exception: SFSR[%016lx] "
161 "SFAR[%016lx], going.\n", sfsr, sfar);
162 die_if_kernel("Iax", regs);
163 }
164 if (test_thread_flag(TIF_32BIT)) {
165 regs->tpc &= 0xffffffff;
166 regs->tnpc &= 0xffffffff;
167 }
168 info.si_signo = SIGSEGV;
169 info.si_errno = 0;
170 info.si_code = SEGV_MAPERR;
171 info.si_addr = (void __user *)regs->tpc;
172 info.si_trapno = 0;
173 force_sig_info(SIGSEGV, &info, current);
174 }
176 void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
177 {
178 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
179 0, 0x8, SIGTRAP) == NOTIFY_STOP)
180 return;
182 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
183 spitfire_insn_access_exception(regs, sfsr, sfar);
184 }
186 void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
187 {
188 unsigned short type = (type_ctx >> 16);
189 unsigned short ctx = (type_ctx & 0xffff);
190 siginfo_t info;
192 if (notify_die(DIE_TRAP, "instruction access exception", regs,
193 0, 0x8, SIGTRAP) == NOTIFY_STOP)
194 return;
196 if (regs->tstate & TSTATE_PRIV) {
197 printk("sun4v_insn_access_exception: ADDR[%016lx] "
198 "CTX[%04x] TYPE[%04x], going.\n",
199 addr, ctx, type);
200 die_if_kernel("Iax", regs);
201 }
203 if (test_thread_flag(TIF_32BIT)) {
204 regs->tpc &= 0xffffffff;
205 regs->tnpc &= 0xffffffff;
206 }
207 info.si_signo = SIGSEGV;
208 info.si_errno = 0;
209 info.si_code = SEGV_MAPERR;
210 info.si_addr = (void __user *) addr;
211 info.si_trapno = 0;
212 force_sig_info(SIGSEGV, &info, current);
213 }
215 void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
216 {
217 if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs,
218 0, 0x8, SIGTRAP) == NOTIFY_STOP)
219 return;
221 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
222 sun4v_insn_access_exception(regs, addr, type_ctx);
223 }
225 void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
226 {
227 siginfo_t info;
229 if (notify_die(DIE_TRAP, "data access exception", regs,
230 0, 0x30, SIGTRAP) == NOTIFY_STOP)
231 return;
233 if (regs->tstate & TSTATE_PRIV) {
234 /* Test if this comes from uaccess places. */
235 const struct exception_table_entry *entry;
237 entry = search_exception_tables(regs->tpc);
238 if (entry) {
239 /* Ouch, somebody is trying VM hole tricks on us... */
240 #ifdef DEBUG_EXCEPTIONS
241 printk("Exception: PC<%016lx> faddr<UNKNOWN>\n", regs->tpc);
242 printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n",
243 regs->tpc, entry->fixup);
244 #endif
245 regs->tpc = entry->fixup;
246 regs->tnpc = regs->tpc + 4;
247 return;
248 }
249 /* Shit... */
250 printk("spitfire_data_access_exception: SFSR[%016lx] "
251 "SFAR[%016lx], going.\n", sfsr, sfar);
252 die_if_kernel("Dax", regs);
253 }
255 info.si_signo = SIGSEGV;
256 info.si_errno = 0;
257 info.si_code = SEGV_MAPERR;
258 info.si_addr = (void __user *)sfar;
259 info.si_trapno = 0;
260 force_sig_info(SIGSEGV, &info, current);
261 }
263 void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar)
264 {
265 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
266 0, 0x30, SIGTRAP) == NOTIFY_STOP)
267 return;
269 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
270 spitfire_data_access_exception(regs, sfsr, sfar);
271 }
273 void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
274 {
275 unsigned short type = (type_ctx >> 16);
276 unsigned short ctx = (type_ctx & 0xffff);
277 siginfo_t info;
279 if (notify_die(DIE_TRAP, "data access exception", regs,
280 0, 0x8, SIGTRAP) == NOTIFY_STOP)
281 return;
283 if (regs->tstate & TSTATE_PRIV) {
284 printk("sun4v_data_access_exception: ADDR[%016lx] "
285 "CTX[%04x] TYPE[%04x], going.\n",
286 addr, ctx, type);
287 die_if_kernel("Dax", regs);
288 }
290 if (test_thread_flag(TIF_32BIT)) {
291 regs->tpc &= 0xffffffff;
292 regs->tnpc &= 0xffffffff;
293 }
294 info.si_signo = SIGSEGV;
295 info.si_errno = 0;
296 info.si_code = SEGV_MAPERR;
297 info.si_addr = (void __user *) addr;
298 info.si_trapno = 0;
299 force_sig_info(SIGSEGV, &info, current);
300 }
302 void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
303 {
304 if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs,
305 0, 0x8, SIGTRAP) == NOTIFY_STOP)
306 return;
308 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
309 sun4v_data_access_exception(regs, addr, type_ctx);
310 }
312 #ifdef CONFIG_PCI
313 /* This is really pathetic... */
314 extern volatile int pci_poke_in_progress;
315 extern volatile int pci_poke_cpu;
316 extern volatile int pci_poke_faulted;
317 #endif
319 /* When access exceptions happen, we must do this. */
320 static void spitfire_clean_and_reenable_l1_caches(void)
321 {
322 unsigned long va;
324 if (tlb_type != spitfire)
325 BUG();
327 /* Clean 'em. */
328 for (va = 0; va < (PAGE_SIZE << 1); va += 32) {
329 spitfire_put_icache_tag(va, 0x0);
330 spitfire_put_dcache_tag(va, 0x0);
331 }
333 /* Re-enable in LSU. */
334 __asm__ __volatile__("flush %%g6\n\t"
335 "membar #Sync\n\t"
336 "stxa %0, [%%g0] %1\n\t"
337 "membar #Sync"
338 : /* no outputs */
339 : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC |
340 LSU_CONTROL_IM | LSU_CONTROL_DM),
341 "i" (ASI_LSU_CONTROL)
342 : "memory");
343 }
345 static void spitfire_enable_estate_errors(void)
346 {
347 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
348 "membar #Sync"
349 : /* no outputs */
350 : "r" (ESTATE_ERR_ALL),
351 "i" (ASI_ESTATE_ERROR_EN));
352 }
354 static char ecc_syndrome_table[] = {
355 0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49,
356 0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a,
357 0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48,
358 0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c,
359 0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48,
360 0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29,
361 0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b,
362 0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48,
363 0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48,
364 0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e,
365 0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b,
366 0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
367 0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36,
368 0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48,
369 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48,
370 0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
371 0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48,
372 0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b,
373 0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32,
374 0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48,
375 0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b,
376 0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48,
377 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48,
378 0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b,
379 0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49,
380 0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48,
381 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48,
382 0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
383 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48,
384 0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b,
385 0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b,
386 0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a
387 };
389 static char *syndrome_unknown = "<Unknown>";
391 static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit)
392 {
393 unsigned short scode;
394 char memmod_str[64], *p;
396 if (udbl & bit) {
397 scode = ecc_syndrome_table[udbl & 0xff];
398 if (prom_getunumber(scode, afar,
399 memmod_str, sizeof(memmod_str)) == -1)
400 p = syndrome_unknown;
401 else
402 p = memmod_str;
403 printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] "
404 "Memory Module \"%s\"\n",
405 smp_processor_id(), scode, p);
406 }
408 if (udbh & bit) {
409 scode = ecc_syndrome_table[udbh & 0xff];
410 if (prom_getunumber(scode, afar,
411 memmod_str, sizeof(memmod_str)) == -1)
412 p = syndrome_unknown;
413 else
414 p = memmod_str;
415 printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] "
416 "Memory Module \"%s\"\n",
417 smp_processor_id(), scode, p);
418 }
420 }
422 static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs)
423 {
425 printk(KERN_WARNING "CPU[%d]: Correctable ECC Error "
426 "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n",
427 smp_processor_id(), afsr, afar, udbl, udbh, tl1);
429 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE);
431 /* We always log it, even if someone is listening for this
432 * trap.
433 */
434 notify_die(DIE_TRAP, "Correctable ECC Error", regs,
435 0, TRAP_TYPE_CEE, SIGTRAP);
437 /* The Correctable ECC Error trap does not disable I/D caches. So
438 * we only have to restore the ESTATE Error Enable register.
439 */
440 spitfire_enable_estate_errors();
441 }
443 static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs)
444 {
445 siginfo_t info;
447 printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] "
448 "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n",
449 smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1);
451 /* XXX add more human friendly logging of the error status
452 * XXX as is implemented for cheetah
453 */
455 spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE);
457 /* We always log it, even if someone is listening for this
458 * trap.
459 */
460 notify_die(DIE_TRAP, "Uncorrectable Error", regs,
461 0, tt, SIGTRAP);
463 if (regs->tstate & TSTATE_PRIV) {
464 if (tl1)
465 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
466 die_if_kernel("UE", regs);
467 }
469 /* XXX need more intelligent processing here, such as is implemented
470 * XXX for cheetah errors, in fact if the E-cache still holds the
471 * XXX line with bad parity this will loop
472 */
474 spitfire_clean_and_reenable_l1_caches();
475 spitfire_enable_estate_errors();
477 if (test_thread_flag(TIF_32BIT)) {
478 regs->tpc &= 0xffffffff;
479 regs->tnpc &= 0xffffffff;
480 }
481 info.si_signo = SIGBUS;
482 info.si_errno = 0;
483 info.si_code = BUS_OBJERR;
484 info.si_addr = (void *)0;
485 info.si_trapno = 0;
486 force_sig_info(SIGBUS, &info, current);
487 }
489 void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar)
490 {
491 unsigned long afsr, tt, udbh, udbl;
492 int tl1;
494 afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT;
495 tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT;
496 tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0;
497 udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT;
498 udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT;
500 #ifdef CONFIG_PCI
501 if (tt == TRAP_TYPE_DAE &&
502 pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
503 spitfire_clean_and_reenable_l1_caches();
504 spitfire_enable_estate_errors();
506 pci_poke_faulted = 1;
507 regs->tnpc = regs->tpc + 4;
508 return;
509 }
510 #endif
512 if (afsr & SFAFSR_UE)
513 spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs);
515 if (tt == TRAP_TYPE_CEE) {
516 /* Handle the case where we took a CEE trap, but ACK'd
517 * only the UE state in the UDB error registers.
518 */
519 if (afsr & SFAFSR_UE) {
520 if (udbh & UDBE_CE) {
521 __asm__ __volatile__(
522 "stxa %0, [%1] %2\n\t"
523 "membar #Sync"
524 : /* no outputs */
525 : "r" (udbh & UDBE_CE),
526 "r" (0x0), "i" (ASI_UDB_ERROR_W));
527 }
528 if (udbl & UDBE_CE) {
529 __asm__ __volatile__(
530 "stxa %0, [%1] %2\n\t"
531 "membar #Sync"
532 : /* no outputs */
533 : "r" (udbl & UDBE_CE),
534 "r" (0x18), "i" (ASI_UDB_ERROR_W));
535 }
536 }
538 spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs);
539 }
540 }
542 int cheetah_pcache_forced_on;
544 void cheetah_enable_pcache(void)
545 {
546 unsigned long dcr;
548 printk("CHEETAH: Enabling P-Cache on cpu %d.\n",
549 smp_processor_id());
551 __asm__ __volatile__("ldxa [%%g0] %1, %0"
552 : "=r" (dcr)
553 : "i" (ASI_DCU_CONTROL_REG));
554 dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL);
555 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
556 "membar #Sync"
557 : /* no outputs */
558 : "r" (dcr), "i" (ASI_DCU_CONTROL_REG));
559 }
561 /* Cheetah error trap handling. */
562 static unsigned long ecache_flush_physbase;
563 static unsigned long ecache_flush_linesize;
564 static unsigned long ecache_flush_size;
566 /* WARNING: The error trap handlers in assembly know the precise
567 * layout of the following structure.
568 *
569 * C-level handlers below use this information to log the error
570 * and then determine how to recover (if possible).
571 */
572 struct cheetah_err_info {
573 /*0x00*/u64 afsr;
574 /*0x08*/u64 afar;
576 /* D-cache state */
577 /*0x10*/u64 dcache_data[4]; /* The actual data */
578 /*0x30*/u64 dcache_index; /* D-cache index */
579 /*0x38*/u64 dcache_tag; /* D-cache tag/valid */
580 /*0x40*/u64 dcache_utag; /* D-cache microtag */
581 /*0x48*/u64 dcache_stag; /* D-cache snooptag */
583 /* I-cache state */
584 /*0x50*/u64 icache_data[8]; /* The actual insns + predecode */
585 /*0x90*/u64 icache_index; /* I-cache index */
586 /*0x98*/u64 icache_tag; /* I-cache phys tag */
587 /*0xa0*/u64 icache_utag; /* I-cache microtag */
588 /*0xa8*/u64 icache_stag; /* I-cache snooptag */
589 /*0xb0*/u64 icache_upper; /* I-cache upper-tag */
590 /*0xb8*/u64 icache_lower; /* I-cache lower-tag */
592 /* E-cache state */
593 /*0xc0*/u64 ecache_data[4]; /* 32 bytes from staging registers */
594 /*0xe0*/u64 ecache_index; /* E-cache index */
595 /*0xe8*/u64 ecache_tag; /* E-cache tag/state */
597 /*0xf0*/u64 __pad[32 - 30];
598 };
599 #define CHAFSR_INVALID ((u64)-1L)
601 /* This table is ordered in priority of errors and matches the
602 * AFAR overwrite policy as well.
603 */
605 struct afsr_error_table {
606 unsigned long mask;
607 const char *name;
608 };
610 static const char CHAFSR_PERR_msg[] =
611 "System interface protocol error";
612 static const char CHAFSR_IERR_msg[] =
613 "Internal processor error";
614 static const char CHAFSR_ISAP_msg[] =
615 "System request parity error on incoming addresss";
616 static const char CHAFSR_UCU_msg[] =
617 "Uncorrectable E-cache ECC error for ifetch/data";
618 static const char CHAFSR_UCC_msg[] =
619 "SW Correctable E-cache ECC error for ifetch/data";
620 static const char CHAFSR_UE_msg[] =
621 "Uncorrectable system bus data ECC error for read";
622 static const char CHAFSR_EDU_msg[] =
623 "Uncorrectable E-cache ECC error for stmerge/blkld";
624 static const char CHAFSR_EMU_msg[] =
625 "Uncorrectable system bus MTAG error";
626 static const char CHAFSR_WDU_msg[] =
627 "Uncorrectable E-cache ECC error for writeback";
628 static const char CHAFSR_CPU_msg[] =
629 "Uncorrectable ECC error for copyout";
630 static const char CHAFSR_CE_msg[] =
631 "HW corrected system bus data ECC error for read";
632 static const char CHAFSR_EDC_msg[] =
633 "HW corrected E-cache ECC error for stmerge/blkld";
634 static const char CHAFSR_EMC_msg[] =
635 "HW corrected system bus MTAG ECC error";
636 static const char CHAFSR_WDC_msg[] =
637 "HW corrected E-cache ECC error for writeback";
638 static const char CHAFSR_CPC_msg[] =
639 "HW corrected ECC error for copyout";
640 static const char CHAFSR_TO_msg[] =
641 "Unmapped error from system bus";
642 static const char CHAFSR_BERR_msg[] =
643 "Bus error response from system bus";
644 static const char CHAFSR_IVC_msg[] =
645 "HW corrected system bus data ECC error for ivec read";
646 static const char CHAFSR_IVU_msg[] =
647 "Uncorrectable system bus data ECC error for ivec read";
648 static struct afsr_error_table __cheetah_error_table[] = {
649 { CHAFSR_PERR, CHAFSR_PERR_msg },
650 { CHAFSR_IERR, CHAFSR_IERR_msg },
651 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
652 { CHAFSR_UCU, CHAFSR_UCU_msg },
653 { CHAFSR_UCC, CHAFSR_UCC_msg },
654 { CHAFSR_UE, CHAFSR_UE_msg },
655 { CHAFSR_EDU, CHAFSR_EDU_msg },
656 { CHAFSR_EMU, CHAFSR_EMU_msg },
657 { CHAFSR_WDU, CHAFSR_WDU_msg },
658 { CHAFSR_CPU, CHAFSR_CPU_msg },
659 { CHAFSR_CE, CHAFSR_CE_msg },
660 { CHAFSR_EDC, CHAFSR_EDC_msg },
661 { CHAFSR_EMC, CHAFSR_EMC_msg },
662 { CHAFSR_WDC, CHAFSR_WDC_msg },
663 { CHAFSR_CPC, CHAFSR_CPC_msg },
664 { CHAFSR_TO, CHAFSR_TO_msg },
665 { CHAFSR_BERR, CHAFSR_BERR_msg },
666 /* These two do not update the AFAR. */
667 { CHAFSR_IVC, CHAFSR_IVC_msg },
668 { CHAFSR_IVU, CHAFSR_IVU_msg },
669 { 0, NULL },
670 };
671 static const char CHPAFSR_DTO_msg[] =
672 "System bus unmapped error for prefetch/storequeue-read";
673 static const char CHPAFSR_DBERR_msg[] =
674 "System bus error for prefetch/storequeue-read";
675 static const char CHPAFSR_THCE_msg[] =
676 "Hardware corrected E-cache Tag ECC error";
677 static const char CHPAFSR_TSCE_msg[] =
678 "SW handled correctable E-cache Tag ECC error";
679 static const char CHPAFSR_TUE_msg[] =
680 "Uncorrectable E-cache Tag ECC error";
681 static const char CHPAFSR_DUE_msg[] =
682 "System bus uncorrectable data ECC error due to prefetch/store-fill";
683 static struct afsr_error_table __cheetah_plus_error_table[] = {
684 { CHAFSR_PERR, CHAFSR_PERR_msg },
685 { CHAFSR_IERR, CHAFSR_IERR_msg },
686 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
687 { CHAFSR_UCU, CHAFSR_UCU_msg },
688 { CHAFSR_UCC, CHAFSR_UCC_msg },
689 { CHAFSR_UE, CHAFSR_UE_msg },
690 { CHAFSR_EDU, CHAFSR_EDU_msg },
691 { CHAFSR_EMU, CHAFSR_EMU_msg },
692 { CHAFSR_WDU, CHAFSR_WDU_msg },
693 { CHAFSR_CPU, CHAFSR_CPU_msg },
694 { CHAFSR_CE, CHAFSR_CE_msg },
695 { CHAFSR_EDC, CHAFSR_EDC_msg },
696 { CHAFSR_EMC, CHAFSR_EMC_msg },
697 { CHAFSR_WDC, CHAFSR_WDC_msg },
698 { CHAFSR_CPC, CHAFSR_CPC_msg },
699 { CHAFSR_TO, CHAFSR_TO_msg },
700 { CHAFSR_BERR, CHAFSR_BERR_msg },
701 { CHPAFSR_DTO, CHPAFSR_DTO_msg },
702 { CHPAFSR_DBERR, CHPAFSR_DBERR_msg },
703 { CHPAFSR_THCE, CHPAFSR_THCE_msg },
704 { CHPAFSR_TSCE, CHPAFSR_TSCE_msg },
705 { CHPAFSR_TUE, CHPAFSR_TUE_msg },
706 { CHPAFSR_DUE, CHPAFSR_DUE_msg },
707 /* These two do not update the AFAR. */
708 { CHAFSR_IVC, CHAFSR_IVC_msg },
709 { CHAFSR_IVU, CHAFSR_IVU_msg },
710 { 0, NULL },
711 };
712 static const char JPAFSR_JETO_msg[] =
713 "System interface protocol error, hw timeout caused";
714 static const char JPAFSR_SCE_msg[] =
715 "Parity error on system snoop results";
716 static const char JPAFSR_JEIC_msg[] =
717 "System interface protocol error, illegal command detected";
718 static const char JPAFSR_JEIT_msg[] =
719 "System interface protocol error, illegal ADTYPE detected";
720 static const char JPAFSR_OM_msg[] =
721 "Out of range memory error has occurred";
722 static const char JPAFSR_ETP_msg[] =
723 "Parity error on L2 cache tag SRAM";
724 static const char JPAFSR_UMS_msg[] =
725 "Error due to unsupported store";
726 static const char JPAFSR_RUE_msg[] =
727 "Uncorrectable ECC error from remote cache/memory";
728 static const char JPAFSR_RCE_msg[] =
729 "Correctable ECC error from remote cache/memory";
730 static const char JPAFSR_BP_msg[] =
731 "JBUS parity error on returned read data";
732 static const char JPAFSR_WBP_msg[] =
733 "JBUS parity error on data for writeback or block store";
734 static const char JPAFSR_FRC_msg[] =
735 "Foreign read to DRAM incurring correctable ECC error";
736 static const char JPAFSR_FRU_msg[] =
737 "Foreign read to DRAM incurring uncorrectable ECC error";
738 static struct afsr_error_table __jalapeno_error_table[] = {
739 { JPAFSR_JETO, JPAFSR_JETO_msg },
740 { JPAFSR_SCE, JPAFSR_SCE_msg },
741 { JPAFSR_JEIC, JPAFSR_JEIC_msg },
742 { JPAFSR_JEIT, JPAFSR_JEIT_msg },
743 { CHAFSR_PERR, CHAFSR_PERR_msg },
744 { CHAFSR_IERR, CHAFSR_IERR_msg },
745 { CHAFSR_ISAP, CHAFSR_ISAP_msg },
746 { CHAFSR_UCU, CHAFSR_UCU_msg },
747 { CHAFSR_UCC, CHAFSR_UCC_msg },
748 { CHAFSR_UE, CHAFSR_UE_msg },
749 { CHAFSR_EDU, CHAFSR_EDU_msg },
750 { JPAFSR_OM, JPAFSR_OM_msg },
751 { CHAFSR_WDU, CHAFSR_WDU_msg },
752 { CHAFSR_CPU, CHAFSR_CPU_msg },
753 { CHAFSR_CE, CHAFSR_CE_msg },
754 { CHAFSR_EDC, CHAFSR_EDC_msg },
755 { JPAFSR_ETP, JPAFSR_ETP_msg },
756 { CHAFSR_WDC, CHAFSR_WDC_msg },
757 { CHAFSR_CPC, CHAFSR_CPC_msg },
758 { CHAFSR_TO, CHAFSR_TO_msg },
759 { CHAFSR_BERR, CHAFSR_BERR_msg },
760 { JPAFSR_UMS, JPAFSR_UMS_msg },
761 { JPAFSR_RUE, JPAFSR_RUE_msg },
762 { JPAFSR_RCE, JPAFSR_RCE_msg },
763 { JPAFSR_BP, JPAFSR_BP_msg },
764 { JPAFSR_WBP, JPAFSR_WBP_msg },
765 { JPAFSR_FRC, JPAFSR_FRC_msg },
766 { JPAFSR_FRU, JPAFSR_FRU_msg },
767 /* These two do not update the AFAR. */
768 { CHAFSR_IVU, CHAFSR_IVU_msg },
769 { 0, NULL },
770 };
771 static struct afsr_error_table *cheetah_error_table;
772 static unsigned long cheetah_afsr_errors;
774 /* This is allocated at boot time based upon the largest hardware
775 * cpu ID in the system. We allocate two entries per cpu, one for
776 * TL==0 logging and one for TL >= 1 logging.
777 */
778 struct cheetah_err_info *cheetah_error_log;
780 static __inline__ struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr)
781 {
782 struct cheetah_err_info *p;
783 int cpu = smp_processor_id();
785 if (!cheetah_error_log)
786 return NULL;
788 p = cheetah_error_log + (cpu * 2);
789 if ((afsr & CHAFSR_TL1) != 0UL)
790 p++;
792 return p;
793 }
795 extern unsigned int tl0_icpe[], tl1_icpe[];
796 extern unsigned int tl0_dcpe[], tl1_dcpe[];
797 extern unsigned int tl0_fecc[], tl1_fecc[];
798 extern unsigned int tl0_cee[], tl1_cee[];
799 extern unsigned int tl0_iae[], tl1_iae[];
800 extern unsigned int tl0_dae[], tl1_dae[];
801 extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[];
802 extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[];
803 extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[];
804 extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[];
805 extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[];
807 void __init cheetah_ecache_flush_init(void)
808 {
809 unsigned long largest_size, smallest_linesize, order, ver;
810 struct device_node *dp;
811 int i, instance, sz;
813 /* Scan all cpu device tree nodes, note two values:
814 * 1) largest E-cache size
815 * 2) smallest E-cache line size
816 */
817 largest_size = 0UL;
818 smallest_linesize = ~0UL;
820 instance = 0;
821 while (!cpu_find_by_instance(instance, &dp, NULL)) {
822 unsigned long val;
824 val = of_getintprop_default(dp, "ecache-size",
825 (2 * 1024 * 1024));
826 if (val > largest_size)
827 largest_size = val;
828 val = of_getintprop_default(dp, "ecache-line-size", 64);
829 if (val < smallest_linesize)
830 smallest_linesize = val;
831 instance++;
832 }
834 if (largest_size == 0UL || smallest_linesize == ~0UL) {
835 prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache "
836 "parameters.\n");
837 prom_halt();
838 }
840 ecache_flush_size = (2 * largest_size);
841 ecache_flush_linesize = smallest_linesize;
843 ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size);
845 if (ecache_flush_physbase == ~0UL) {
846 prom_printf("cheetah_ecache_flush_init: Cannot find %d byte "
847 "contiguous physical memory.\n",
848 ecache_flush_size);
849 prom_halt();
850 }
852 /* Now allocate error trap reporting scoreboard. */
853 sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info));
854 for (order = 0; order < MAX_ORDER; order++) {
855 if ((PAGE_SIZE << order) >= sz)
856 break;
857 }
858 cheetah_error_log = (struct cheetah_err_info *)
859 __get_free_pages(GFP_KERNEL, order);
860 if (!cheetah_error_log) {
861 prom_printf("cheetah_ecache_flush_init: Failed to allocate "
862 "error logging scoreboard (%d bytes).\n", sz);
863 prom_halt();
864 }
865 memset(cheetah_error_log, 0, PAGE_SIZE << order);
867 /* Mark all AFSRs as invalid so that the trap handler will
868 * log new new information there.
869 */
870 for (i = 0; i < 2 * NR_CPUS; i++)
871 cheetah_error_log[i].afsr = CHAFSR_INVALID;
873 __asm__ ("rdpr %%ver, %0" : "=r" (ver));
874 if ((ver >> 32) == __JALAPENO_ID ||
875 (ver >> 32) == __SERRANO_ID) {
876 cheetah_error_table = &__jalapeno_error_table[0];
877 cheetah_afsr_errors = JPAFSR_ERRORS;
878 } else if ((ver >> 32) == 0x003e0015) {
879 cheetah_error_table = &__cheetah_plus_error_table[0];
880 cheetah_afsr_errors = CHPAFSR_ERRORS;
881 } else {
882 cheetah_error_table = &__cheetah_error_table[0];
883 cheetah_afsr_errors = CHAFSR_ERRORS;
884 }
886 /* Now patch trap tables. */
887 memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4));
888 memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4));
889 memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4));
890 memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4));
891 memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4));
892 memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4));
893 memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4));
894 memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4));
895 if (tlb_type == cheetah_plus) {
896 memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4));
897 memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4));
898 memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4));
899 memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4));
900 }
901 flushi(PAGE_OFFSET);
902 }
904 static void cheetah_flush_ecache(void)
905 {
906 unsigned long flush_base = ecache_flush_physbase;
907 unsigned long flush_linesize = ecache_flush_linesize;
908 unsigned long flush_size = ecache_flush_size;
910 __asm__ __volatile__("1: subcc %0, %4, %0\n\t"
911 " bne,pt %%xcc, 1b\n\t"
912 " ldxa [%2 + %0] %3, %%g0\n\t"
913 : "=&r" (flush_size)
914 : "0" (flush_size), "r" (flush_base),
915 "i" (ASI_PHYS_USE_EC), "r" (flush_linesize));
916 }
918 static void cheetah_flush_ecache_line(unsigned long physaddr)
919 {
920 unsigned long alias;
922 physaddr &= ~(8UL - 1UL);
923 physaddr = (ecache_flush_physbase +
924 (physaddr & ((ecache_flush_size>>1UL) - 1UL)));
925 alias = physaddr + (ecache_flush_size >> 1UL);
926 __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t"
927 "ldxa [%1] %2, %%g0\n\t"
928 "membar #Sync"
929 : /* no outputs */
930 : "r" (physaddr), "r" (alias),
931 "i" (ASI_PHYS_USE_EC));
932 }
934 /* Unfortunately, the diagnostic access to the I-cache tags we need to
935 * use to clear the thing interferes with I-cache coherency transactions.
936 *
937 * So we must only flush the I-cache when it is disabled.
938 */
939 static void __cheetah_flush_icache(void)
940 {
941 unsigned int icache_size, icache_line_size;
942 unsigned long addr;
944 icache_size = local_cpu_data().icache_size;
945 icache_line_size = local_cpu_data().icache_line_size;
947 /* Clear the valid bits in all the tags. */
948 for (addr = 0; addr < icache_size; addr += icache_line_size) {
949 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
950 "membar #Sync"
951 : /* no outputs */
952 : "r" (addr | (2 << 3)),
953 "i" (ASI_IC_TAG));
954 }
955 }
957 static void cheetah_flush_icache(void)
958 {
959 unsigned long dcu_save;
961 /* Save current DCU, disable I-cache. */
962 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
963 "or %0, %2, %%g1\n\t"
964 "stxa %%g1, [%%g0] %1\n\t"
965 "membar #Sync"
966 : "=r" (dcu_save)
967 : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC)
968 : "g1");
970 __cheetah_flush_icache();
972 /* Restore DCU register */
973 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
974 "membar #Sync"
975 : /* no outputs */
976 : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG));
977 }
979 static void cheetah_flush_dcache(void)
980 {
981 unsigned int dcache_size, dcache_line_size;
982 unsigned long addr;
984 dcache_size = local_cpu_data().dcache_size;
985 dcache_line_size = local_cpu_data().dcache_line_size;
987 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
988 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
989 "membar #Sync"
990 : /* no outputs */
991 : "r" (addr), "i" (ASI_DCACHE_TAG));
992 }
993 }
995 /* In order to make the even parity correct we must do two things.
996 * First, we clear DC_data_parity and set DC_utag to an appropriate value.
997 * Next, we clear out all 32-bytes of data for that line. Data of
998 * all-zero + tag parity value of zero == correct parity.
999 */
1000 static void cheetah_plus_zap_dcache_parity(void)
1002 unsigned int dcache_size, dcache_line_size;
1003 unsigned long addr;
1005 dcache_size = local_cpu_data().dcache_size;
1006 dcache_line_size = local_cpu_data().dcache_line_size;
1008 for (addr = 0; addr < dcache_size; addr += dcache_line_size) {
1009 unsigned long tag = (addr >> 14);
1010 unsigned long line;
1012 __asm__ __volatile__("membar #Sync\n\t"
1013 "stxa %0, [%1] %2\n\t"
1014 "membar #Sync"
1015 : /* no outputs */
1016 : "r" (tag), "r" (addr),
1017 "i" (ASI_DCACHE_UTAG));
1018 for (line = addr; line < addr + dcache_line_size; line += 8)
1019 __asm__ __volatile__("membar #Sync\n\t"
1020 "stxa %%g0, [%0] %1\n\t"
1021 "membar #Sync"
1022 : /* no outputs */
1023 : "r" (line),
1024 "i" (ASI_DCACHE_DATA));
1028 /* Conversion tables used to frob Cheetah AFSR syndrome values into
1029 * something palatable to the memory controller driver get_unumber
1030 * routine.
1031 */
1032 #define MT0 137
1033 #define MT1 138
1034 #define MT2 139
1035 #define NONE 254
1036 #define MTC0 140
1037 #define MTC1 141
1038 #define MTC2 142
1039 #define MTC3 143
1040 #define C0 128
1041 #define C1 129
1042 #define C2 130
1043 #define C3 131
1044 #define C4 132
1045 #define C5 133
1046 #define C6 134
1047 #define C7 135
1048 #define C8 136
1049 #define M2 144
1050 #define M3 145
1051 #define M4 146
1052 #define M 147
1053 static unsigned char cheetah_ecc_syntab[] = {
1054 /*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M,
1055 /*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16,
1056 /*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10,
1057 /*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M,
1058 /*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6,
1059 /*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4,
1060 /*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4,
1061 /*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3,
1062 /*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5,
1063 /*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M,
1064 /*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2,
1065 /*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3,
1066 /*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M,
1067 /*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3,
1068 /*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M,
1069 /*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M,
1070 /*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4,
1071 /*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M,
1072 /*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2,
1073 /*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M,
1074 /*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4,
1075 /*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3,
1076 /*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3,
1077 /*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2,
1078 /*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4,
1079 /*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M,
1080 /*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3,
1081 /*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M,
1082 /*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3,
1083 /*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M,
1084 /*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M,
1085 /*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M
1086 };
1087 static unsigned char cheetah_mtag_syntab[] = {
1088 NONE, MTC0,
1089 MTC1, NONE,
1090 MTC2, NONE,
1091 NONE, MT0,
1092 MTC3, NONE,
1093 NONE, MT1,
1094 NONE, MT2,
1095 NONE, NONE
1096 };
1098 /* Return the highest priority error conditon mentioned. */
1099 static __inline__ unsigned long cheetah_get_hipri(unsigned long afsr)
1101 unsigned long tmp = 0;
1102 int i;
1104 for (i = 0; cheetah_error_table[i].mask; i++) {
1105 if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL)
1106 return tmp;
1108 return tmp;
1111 static const char *cheetah_get_string(unsigned long bit)
1113 int i;
1115 for (i = 0; cheetah_error_table[i].mask; i++) {
1116 if ((bit & cheetah_error_table[i].mask) != 0UL)
1117 return cheetah_error_table[i].name;
1119 return "???";
1122 extern int chmc_getunumber(int, unsigned long, char *, int);
1124 static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info,
1125 unsigned long afsr, unsigned long afar, int recoverable)
1127 unsigned long hipri;
1128 char unum[256];
1130 printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n",
1131 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1132 afsr, afar,
1133 (afsr & CHAFSR_TL1) ? 1 : 0);
1134 printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n",
1135 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1136 regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate);
1137 printk("%s" "ERROR(%d): M_SYND(%lx), E_SYND(%lx)%s%s\n",
1138 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1139 (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT,
1140 (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT,
1141 (afsr & CHAFSR_ME) ? ", Multiple Errors" : "",
1142 (afsr & CHAFSR_PRIV) ? ", Privileged" : "");
1143 hipri = cheetah_get_hipri(afsr);
1144 printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n",
1145 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1146 hipri, cheetah_get_string(hipri));
1148 /* Try to get unumber if relevant. */
1149 #define ESYND_ERRORS (CHAFSR_IVC | CHAFSR_IVU | \
1150 CHAFSR_CPC | CHAFSR_CPU | \
1151 CHAFSR_UE | CHAFSR_CE | \
1152 CHAFSR_EDC | CHAFSR_EDU | \
1153 CHAFSR_UCC | CHAFSR_UCU | \
1154 CHAFSR_WDU | CHAFSR_WDC)
1155 #define MSYND_ERRORS (CHAFSR_EMC | CHAFSR_EMU)
1156 if (afsr & ESYND_ERRORS) {
1157 int syndrome;
1158 int ret;
1160 syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT;
1161 syndrome = cheetah_ecc_syntab[syndrome];
1162 ret = chmc_getunumber(syndrome, afar, unum, sizeof(unum));
1163 if (ret != -1)
1164 printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n",
1165 (recoverable ? KERN_WARNING : KERN_CRIT),
1166 smp_processor_id(), unum);
1167 } else if (afsr & MSYND_ERRORS) {
1168 int syndrome;
1169 int ret;
1171 syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT;
1172 syndrome = cheetah_mtag_syntab[syndrome];
1173 ret = chmc_getunumber(syndrome, afar, unum, sizeof(unum));
1174 if (ret != -1)
1175 printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n",
1176 (recoverable ? KERN_WARNING : KERN_CRIT),
1177 smp_processor_id(), unum);
1180 /* Now dump the cache snapshots. */
1181 printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016lx] utag[%016lx] stag[%016lx]\n",
1182 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1183 (int) info->dcache_index,
1184 info->dcache_tag,
1185 info->dcache_utag,
1186 info->dcache_stag);
1187 printk("%s" "ERROR(%d): D-cache data0[%016lx] data1[%016lx] data2[%016lx] data3[%016lx]\n",
1188 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1189 info->dcache_data[0],
1190 info->dcache_data[1],
1191 info->dcache_data[2],
1192 info->dcache_data[3]);
1193 printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016lx] utag[%016lx] stag[%016lx] "
1194 "u[%016lx] l[%016lx]\n",
1195 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1196 (int) info->icache_index,
1197 info->icache_tag,
1198 info->icache_utag,
1199 info->icache_stag,
1200 info->icache_upper,
1201 info->icache_lower);
1202 printk("%s" "ERROR(%d): I-cache INSN0[%016lx] INSN1[%016lx] INSN2[%016lx] INSN3[%016lx]\n",
1203 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1204 info->icache_data[0],
1205 info->icache_data[1],
1206 info->icache_data[2],
1207 info->icache_data[3]);
1208 printk("%s" "ERROR(%d): I-cache INSN4[%016lx] INSN5[%016lx] INSN6[%016lx] INSN7[%016lx]\n",
1209 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1210 info->icache_data[4],
1211 info->icache_data[5],
1212 info->icache_data[6],
1213 info->icache_data[7]);
1214 printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016lx]\n",
1215 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1216 (int) info->ecache_index, info->ecache_tag);
1217 printk("%s" "ERROR(%d): E-cache data0[%016lx] data1[%016lx] data2[%016lx] data3[%016lx]\n",
1218 (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(),
1219 info->ecache_data[0],
1220 info->ecache_data[1],
1221 info->ecache_data[2],
1222 info->ecache_data[3]);
1224 afsr = (afsr & ~hipri) & cheetah_afsr_errors;
1225 while (afsr != 0UL) {
1226 unsigned long bit = cheetah_get_hipri(afsr);
1228 printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n",
1229 (recoverable ? KERN_WARNING : KERN_CRIT),
1230 bit, cheetah_get_string(bit));
1232 afsr &= ~bit;
1235 if (!recoverable)
1236 printk(KERN_CRIT "ERROR: This condition is not recoverable.\n");
1239 static int cheetah_recheck_errors(struct cheetah_err_info *logp)
1241 unsigned long afsr, afar;
1242 int ret = 0;
1244 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1245 : "=r" (afsr)
1246 : "i" (ASI_AFSR));
1247 if ((afsr & cheetah_afsr_errors) != 0) {
1248 if (logp != NULL) {
1249 __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t"
1250 : "=r" (afar)
1251 : "i" (ASI_AFAR));
1252 logp->afsr = afsr;
1253 logp->afar = afar;
1255 ret = 1;
1257 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1258 "membar #Sync\n\t"
1259 : : "r" (afsr), "i" (ASI_AFSR));
1261 return ret;
1264 void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1266 struct cheetah_err_info local_snapshot, *p;
1267 int recoverable;
1269 /* Flush E-cache */
1270 cheetah_flush_ecache();
1272 p = cheetah_get_error_log(afsr);
1273 if (!p) {
1274 prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n",
1275 afsr, afar);
1276 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1277 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1278 prom_halt();
1281 /* Grab snapshot of logged error. */
1282 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1284 /* If the current trap snapshot does not match what the
1285 * trap handler passed along into our args, big trouble.
1286 * In such a case, mark the local copy as invalid.
1288 * Else, it matches and we mark the afsr in the non-local
1289 * copy as invalid so we may log new error traps there.
1290 */
1291 if (p->afsr != afsr || p->afar != afar)
1292 local_snapshot.afsr = CHAFSR_INVALID;
1293 else
1294 p->afsr = CHAFSR_INVALID;
1296 cheetah_flush_icache();
1297 cheetah_flush_dcache();
1299 /* Re-enable I-cache/D-cache */
1300 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1301 "or %%g1, %1, %%g1\n\t"
1302 "stxa %%g1, [%%g0] %0\n\t"
1303 "membar #Sync"
1304 : /* no outputs */
1305 : "i" (ASI_DCU_CONTROL_REG),
1306 "i" (DCU_DC | DCU_IC)
1307 : "g1");
1309 /* Re-enable error reporting */
1310 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1311 "or %%g1, %1, %%g1\n\t"
1312 "stxa %%g1, [%%g0] %0\n\t"
1313 "membar #Sync"
1314 : /* no outputs */
1315 : "i" (ASI_ESTATE_ERROR_EN),
1316 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1317 : "g1");
1319 /* Decide if we can continue after handling this trap and
1320 * logging the error.
1321 */
1322 recoverable = 1;
1323 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1324 recoverable = 0;
1326 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1327 * error was logged while we had error reporting traps disabled.
1328 */
1329 if (cheetah_recheck_errors(&local_snapshot)) {
1330 unsigned long new_afsr = local_snapshot.afsr;
1332 /* If we got a new asynchronous error, die... */
1333 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1334 CHAFSR_WDU | CHAFSR_CPU |
1335 CHAFSR_IVU | CHAFSR_UE |
1336 CHAFSR_BERR | CHAFSR_TO))
1337 recoverable = 0;
1340 /* Log errors. */
1341 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1343 if (!recoverable)
1344 panic("Irrecoverable Fast-ECC error trap.\n");
1346 /* Flush E-cache to kick the error trap handlers out. */
1347 cheetah_flush_ecache();
1350 /* Try to fix a correctable error by pushing the line out from
1351 * the E-cache. Recheck error reporting registers to see if the
1352 * problem is intermittent.
1353 */
1354 static int cheetah_fix_ce(unsigned long physaddr)
1356 unsigned long orig_estate;
1357 unsigned long alias1, alias2;
1358 int ret;
1360 /* Make sure correctable error traps are disabled. */
1361 __asm__ __volatile__("ldxa [%%g0] %2, %0\n\t"
1362 "andn %0, %1, %%g1\n\t"
1363 "stxa %%g1, [%%g0] %2\n\t"
1364 "membar #Sync"
1365 : "=&r" (orig_estate)
1366 : "i" (ESTATE_ERROR_CEEN),
1367 "i" (ASI_ESTATE_ERROR_EN)
1368 : "g1");
1370 /* We calculate alias addresses that will force the
1371 * cache line in question out of the E-cache. Then
1372 * we bring it back in with an atomic instruction so
1373 * that we get it in some modified/exclusive state,
1374 * then we displace it again to try and get proper ECC
1375 * pushed back into the system.
1376 */
1377 physaddr &= ~(8UL - 1UL);
1378 alias1 = (ecache_flush_physbase +
1379 (physaddr & ((ecache_flush_size >> 1) - 1)));
1380 alias2 = alias1 + (ecache_flush_size >> 1);
1381 __asm__ __volatile__("ldxa [%0] %3, %%g0\n\t"
1382 "ldxa [%1] %3, %%g0\n\t"
1383 "casxa [%2] %3, %%g0, %%g0\n\t"
1384 "membar #StoreLoad | #StoreStore\n\t"
1385 "ldxa [%0] %3, %%g0\n\t"
1386 "ldxa [%1] %3, %%g0\n\t"
1387 "membar #Sync"
1388 : /* no outputs */
1389 : "r" (alias1), "r" (alias2),
1390 "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1392 /* Did that trigger another error? */
1393 if (cheetah_recheck_errors(NULL)) {
1394 /* Try one more time. */
1395 __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t"
1396 "membar #Sync"
1397 : : "r" (physaddr), "i" (ASI_PHYS_USE_EC));
1398 if (cheetah_recheck_errors(NULL))
1399 ret = 2;
1400 else
1401 ret = 1;
1402 } else {
1403 /* No new error, intermittent problem. */
1404 ret = 0;
1407 /* Restore error enables. */
1408 __asm__ __volatile__("stxa %0, [%%g0] %1\n\t"
1409 "membar #Sync"
1410 : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN));
1412 return ret;
1415 /* Return non-zero if PADDR is a valid physical memory address. */
1416 static int cheetah_check_main_memory(unsigned long paddr)
1418 unsigned long vaddr = PAGE_OFFSET + paddr;
1420 if (vaddr > (unsigned long) high_memory)
1421 return 0;
1423 return kern_addr_valid(vaddr);
1426 void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1428 struct cheetah_err_info local_snapshot, *p;
1429 int recoverable, is_memory;
1431 p = cheetah_get_error_log(afsr);
1432 if (!p) {
1433 prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n",
1434 afsr, afar);
1435 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1436 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1437 prom_halt();
1440 /* Grab snapshot of logged error. */
1441 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1443 /* If the current trap snapshot does not match what the
1444 * trap handler passed along into our args, big trouble.
1445 * In such a case, mark the local copy as invalid.
1447 * Else, it matches and we mark the afsr in the non-local
1448 * copy as invalid so we may log new error traps there.
1449 */
1450 if (p->afsr != afsr || p->afar != afar)
1451 local_snapshot.afsr = CHAFSR_INVALID;
1452 else
1453 p->afsr = CHAFSR_INVALID;
1455 is_memory = cheetah_check_main_memory(afar);
1457 if (is_memory && (afsr & CHAFSR_CE) != 0UL) {
1458 /* XXX Might want to log the results of this operation
1459 * XXX somewhere... -DaveM
1460 */
1461 cheetah_fix_ce(afar);
1465 int flush_all, flush_line;
1467 flush_all = flush_line = 0;
1468 if ((afsr & CHAFSR_EDC) != 0UL) {
1469 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC)
1470 flush_line = 1;
1471 else
1472 flush_all = 1;
1473 } else if ((afsr & CHAFSR_CPC) != 0UL) {
1474 if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC)
1475 flush_line = 1;
1476 else
1477 flush_all = 1;
1480 /* Trap handler only disabled I-cache, flush it. */
1481 cheetah_flush_icache();
1483 /* Re-enable I-cache */
1484 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1485 "or %%g1, %1, %%g1\n\t"
1486 "stxa %%g1, [%%g0] %0\n\t"
1487 "membar #Sync"
1488 : /* no outputs */
1489 : "i" (ASI_DCU_CONTROL_REG),
1490 "i" (DCU_IC)
1491 : "g1");
1493 if (flush_all)
1494 cheetah_flush_ecache();
1495 else if (flush_line)
1496 cheetah_flush_ecache_line(afar);
1499 /* Re-enable error reporting */
1500 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1501 "or %%g1, %1, %%g1\n\t"
1502 "stxa %%g1, [%%g0] %0\n\t"
1503 "membar #Sync"
1504 : /* no outputs */
1505 : "i" (ASI_ESTATE_ERROR_EN),
1506 "i" (ESTATE_ERROR_CEEN)
1507 : "g1");
1509 /* Decide if we can continue after handling this trap and
1510 * logging the error.
1511 */
1512 recoverable = 1;
1513 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1514 recoverable = 0;
1516 /* Re-check AFSR/AFAR */
1517 (void) cheetah_recheck_errors(&local_snapshot);
1519 /* Log errors. */
1520 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1522 if (!recoverable)
1523 panic("Irrecoverable Correctable-ECC error trap.\n");
1526 void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar)
1528 struct cheetah_err_info local_snapshot, *p;
1529 int recoverable, is_memory;
1531 #ifdef CONFIG_PCI
1532 /* Check for the special PCI poke sequence. */
1533 if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) {
1534 cheetah_flush_icache();
1535 cheetah_flush_dcache();
1537 /* Re-enable I-cache/D-cache */
1538 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1539 "or %%g1, %1, %%g1\n\t"
1540 "stxa %%g1, [%%g0] %0\n\t"
1541 "membar #Sync"
1542 : /* no outputs */
1543 : "i" (ASI_DCU_CONTROL_REG),
1544 "i" (DCU_DC | DCU_IC)
1545 : "g1");
1547 /* Re-enable error reporting */
1548 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1549 "or %%g1, %1, %%g1\n\t"
1550 "stxa %%g1, [%%g0] %0\n\t"
1551 "membar #Sync"
1552 : /* no outputs */
1553 : "i" (ASI_ESTATE_ERROR_EN),
1554 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1555 : "g1");
1557 (void) cheetah_recheck_errors(NULL);
1559 pci_poke_faulted = 1;
1560 regs->tpc += 4;
1561 regs->tnpc = regs->tpc + 4;
1562 return;
1564 #endif
1566 p = cheetah_get_error_log(afsr);
1567 if (!p) {
1568 prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n",
1569 afsr, afar);
1570 prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n",
1571 smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate);
1572 prom_halt();
1575 /* Grab snapshot of logged error. */
1576 memcpy(&local_snapshot, p, sizeof(local_snapshot));
1578 /* If the current trap snapshot does not match what the
1579 * trap handler passed along into our args, big trouble.
1580 * In such a case, mark the local copy as invalid.
1582 * Else, it matches and we mark the afsr in the non-local
1583 * copy as invalid so we may log new error traps there.
1584 */
1585 if (p->afsr != afsr || p->afar != afar)
1586 local_snapshot.afsr = CHAFSR_INVALID;
1587 else
1588 p->afsr = CHAFSR_INVALID;
1590 is_memory = cheetah_check_main_memory(afar);
1593 int flush_all, flush_line;
1595 flush_all = flush_line = 0;
1596 if ((afsr & CHAFSR_EDU) != 0UL) {
1597 if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU)
1598 flush_line = 1;
1599 else
1600 flush_all = 1;
1601 } else if ((afsr & CHAFSR_BERR) != 0UL) {
1602 if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR)
1603 flush_line = 1;
1604 else
1605 flush_all = 1;
1608 cheetah_flush_icache();
1609 cheetah_flush_dcache();
1611 /* Re-enable I/D caches */
1612 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1613 "or %%g1, %1, %%g1\n\t"
1614 "stxa %%g1, [%%g0] %0\n\t"
1615 "membar #Sync"
1616 : /* no outputs */
1617 : "i" (ASI_DCU_CONTROL_REG),
1618 "i" (DCU_IC | DCU_DC)
1619 : "g1");
1621 if (flush_all)
1622 cheetah_flush_ecache();
1623 else if (flush_line)
1624 cheetah_flush_ecache_line(afar);
1627 /* Re-enable error reporting */
1628 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1629 "or %%g1, %1, %%g1\n\t"
1630 "stxa %%g1, [%%g0] %0\n\t"
1631 "membar #Sync"
1632 : /* no outputs */
1633 : "i" (ASI_ESTATE_ERROR_EN),
1634 "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN)
1635 : "g1");
1637 /* Decide if we can continue after handling this trap and
1638 * logging the error.
1639 */
1640 recoverable = 1;
1641 if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP))
1642 recoverable = 0;
1644 /* Re-check AFSR/AFAR. What we are looking for here is whether a new
1645 * error was logged while we had error reporting traps disabled.
1646 */
1647 if (cheetah_recheck_errors(&local_snapshot)) {
1648 unsigned long new_afsr = local_snapshot.afsr;
1650 /* If we got a new asynchronous error, die... */
1651 if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU |
1652 CHAFSR_WDU | CHAFSR_CPU |
1653 CHAFSR_IVU | CHAFSR_UE |
1654 CHAFSR_BERR | CHAFSR_TO))
1655 recoverable = 0;
1658 /* Log errors. */
1659 cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable);
1661 /* "Recoverable" here means we try to yank the page from ever
1662 * being newly used again. This depends upon a few things:
1663 * 1) Must be main memory, and AFAR must be valid.
1664 * 2) If we trapped from user, OK.
1665 * 3) Else, if we trapped from kernel we must find exception
1666 * table entry (ie. we have to have been accessing user
1667 * space).
1669 * If AFAR is not in main memory, or we trapped from kernel
1670 * and cannot find an exception table entry, it is unacceptable
1671 * to try and continue.
1672 */
1673 if (recoverable && is_memory) {
1674 if ((regs->tstate & TSTATE_PRIV) == 0UL) {
1675 /* OK, usermode access. */
1676 recoverable = 1;
1677 } else {
1678 const struct exception_table_entry *entry;
1680 entry = search_exception_tables(regs->tpc);
1681 if (entry) {
1682 /* OK, kernel access to userspace. */
1683 recoverable = 1;
1685 } else {
1686 /* BAD, privileged state is corrupted. */
1687 recoverable = 0;
1690 if (recoverable) {
1691 if (pfn_valid(afar >> PAGE_SHIFT))
1692 get_page(pfn_to_page(afar >> PAGE_SHIFT));
1693 else
1694 recoverable = 0;
1696 /* Only perform fixup if we still have a
1697 * recoverable condition.
1698 */
1699 if (recoverable) {
1700 regs->tpc = entry->fixup;
1701 regs->tnpc = regs->tpc + 4;
1705 } else {
1706 recoverable = 0;
1709 if (!recoverable)
1710 panic("Irrecoverable deferred error trap.\n");
1713 /* Handle a D/I cache parity error trap. TYPE is encoded as:
1715 * Bit0: 0=dcache,1=icache
1716 * Bit1: 0=recoverable,1=unrecoverable
1718 * The hardware has disabled both the I-cache and D-cache in
1719 * the %dcr register.
1720 */
1721 void cheetah_plus_parity_error(int type, struct pt_regs *regs)
1723 if (type & 0x1)
1724 __cheetah_flush_icache();
1725 else
1726 cheetah_plus_zap_dcache_parity();
1727 cheetah_flush_dcache();
1729 /* Re-enable I-cache/D-cache */
1730 __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t"
1731 "or %%g1, %1, %%g1\n\t"
1732 "stxa %%g1, [%%g0] %0\n\t"
1733 "membar #Sync"
1734 : /* no outputs */
1735 : "i" (ASI_DCU_CONTROL_REG),
1736 "i" (DCU_DC | DCU_IC)
1737 : "g1");
1739 if (type & 0x2) {
1740 printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1741 smp_processor_id(),
1742 (type & 0x1) ? 'I' : 'D',
1743 regs->tpc);
1744 panic("Irrecoverable Cheetah+ parity error.");
1747 printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n",
1748 smp_processor_id(),
1749 (type & 0x1) ? 'I' : 'D',
1750 regs->tpc);
1753 struct sun4v_error_entry {
1754 u64 err_handle;
1755 u64 err_stick;
1757 u32 err_type;
1758 #define SUN4V_ERR_TYPE_UNDEFINED 0
1759 #define SUN4V_ERR_TYPE_UNCORRECTED_RES 1
1760 #define SUN4V_ERR_TYPE_PRECISE_NONRES 2
1761 #define SUN4V_ERR_TYPE_DEFERRED_NONRES 3
1762 #define SUN4V_ERR_TYPE_WARNING_RES 4
1764 u32 err_attrs;
1765 #define SUN4V_ERR_ATTRS_PROCESSOR 0x00000001
1766 #define SUN4V_ERR_ATTRS_MEMORY 0x00000002
1767 #define SUN4V_ERR_ATTRS_PIO 0x00000004
1768 #define SUN4V_ERR_ATTRS_INT_REGISTERS 0x00000008
1769 #define SUN4V_ERR_ATTRS_FPU_REGISTERS 0x00000010
1770 #define SUN4V_ERR_ATTRS_USER_MODE 0x01000000
1771 #define SUN4V_ERR_ATTRS_PRIV_MODE 0x02000000
1772 #define SUN4V_ERR_ATTRS_RES_QUEUE_FULL 0x80000000
1774 u64 err_raddr;
1775 u32 err_size;
1776 u16 err_cpu;
1777 u16 err_pad;
1778 };
1780 static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0);
1781 static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0);
1783 static const char *sun4v_err_type_to_str(u32 type)
1785 switch (type) {
1786 case SUN4V_ERR_TYPE_UNDEFINED:
1787 return "undefined";
1788 case SUN4V_ERR_TYPE_UNCORRECTED_RES:
1789 return "uncorrected resumable";
1790 case SUN4V_ERR_TYPE_PRECISE_NONRES:
1791 return "precise nonresumable";
1792 case SUN4V_ERR_TYPE_DEFERRED_NONRES:
1793 return "deferred nonresumable";
1794 case SUN4V_ERR_TYPE_WARNING_RES:
1795 return "warning resumable";
1796 default:
1797 return "unknown";
1798 };
1801 extern void __show_regs(struct pt_regs * regs);
1803 static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, int cpu, const char *pfx, atomic_t *ocnt)
1805 int cnt;
1807 printk("%s: Reporting on cpu %d\n", pfx, cpu);
1808 printk("%s: err_handle[%lx] err_stick[%lx] err_type[%08x:%s]\n",
1809 pfx,
1810 ent->err_handle, ent->err_stick,
1811 ent->err_type,
1812 sun4v_err_type_to_str(ent->err_type));
1813 printk("%s: err_attrs[%08x:%s %s %s %s %s %s %s %s]\n",
1814 pfx,
1815 ent->err_attrs,
1816 ((ent->err_attrs & SUN4V_ERR_ATTRS_PROCESSOR) ?
1817 "processor" : ""),
1818 ((ent->err_attrs & SUN4V_ERR_ATTRS_MEMORY) ?
1819 "memory" : ""),
1820 ((ent->err_attrs & SUN4V_ERR_ATTRS_PIO) ?
1821 "pio" : ""),
1822 ((ent->err_attrs & SUN4V_ERR_ATTRS_INT_REGISTERS) ?
1823 "integer-regs" : ""),
1824 ((ent->err_attrs & SUN4V_ERR_ATTRS_FPU_REGISTERS) ?
1825 "fpu-regs" : ""),
1826 ((ent->err_attrs & SUN4V_ERR_ATTRS_USER_MODE) ?
1827 "user" : ""),
1828 ((ent->err_attrs & SUN4V_ERR_ATTRS_PRIV_MODE) ?
1829 "privileged" : ""),
1830 ((ent->err_attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL) ?
1831 "queue-full" : ""));
1832 printk("%s: err_raddr[%016lx] err_size[%u] err_cpu[%u]\n",
1833 pfx,
1834 ent->err_raddr, ent->err_size, ent->err_cpu);
1836 __show_regs(regs);
1838 if ((cnt = atomic_read(ocnt)) != 0) {
1839 atomic_set(ocnt, 0);
1840 wmb();
1841 printk("%s: Queue overflowed %d times.\n",
1842 pfx, cnt);
1846 /* We run with %pil set to 15 and PSTATE_IE enabled in %pstate.
1847 * Log the event and clear the first word of the entry.
1848 */
1849 void sun4v_resum_error(struct pt_regs *regs, unsigned long offset)
1851 struct sun4v_error_entry *ent, local_copy;
1852 struct trap_per_cpu *tb;
1853 unsigned long paddr;
1854 int cpu;
1856 cpu = get_cpu();
1858 tb = &trap_block[cpu];
1859 paddr = tb->resum_kernel_buf_pa + offset;
1860 ent = __va(paddr);
1862 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
1864 /* We have a local copy now, so release the entry. */
1865 ent->err_handle = 0;
1866 wmb();
1868 put_cpu();
1870 sun4v_log_error(regs, &local_copy, cpu,
1871 KERN_ERR "RESUMABLE ERROR",
1872 &sun4v_resum_oflow_cnt);
1875 /* If we try to printk() we'll probably make matters worse, by trying
1876 * to retake locks this cpu already holds or causing more errors. So
1877 * just bump a counter, and we'll report these counter bumps above.
1878 */
1879 void sun4v_resum_overflow(struct pt_regs *regs)
1881 atomic_inc(&sun4v_resum_oflow_cnt);
1884 /* We run with %pil set to 15 and PSTATE_IE enabled in %pstate.
1885 * Log the event, clear the first word of the entry, and die.
1886 */
1887 void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset)
1889 struct sun4v_error_entry *ent, local_copy;
1890 struct trap_per_cpu *tb;
1891 unsigned long paddr;
1892 int cpu;
1894 cpu = get_cpu();
1896 tb = &trap_block[cpu];
1897 paddr = tb->nonresum_kernel_buf_pa + offset;
1898 ent = __va(paddr);
1900 memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry));
1902 /* We have a local copy now, so release the entry. */
1903 ent->err_handle = 0;
1904 wmb();
1906 put_cpu();
1908 #ifdef CONFIG_PCI
1909 /* Check for the special PCI poke sequence. */
1910 if (pci_poke_in_progress && pci_poke_cpu == cpu) {
1911 pci_poke_faulted = 1;
1912 regs->tpc += 4;
1913 regs->tnpc = regs->tpc + 4;
1914 return;
1916 #endif
1918 sun4v_log_error(regs, &local_copy, cpu,
1919 KERN_EMERG "NON-RESUMABLE ERROR",
1920 &sun4v_nonresum_oflow_cnt);
1922 panic("Non-resumable error.");
1925 /* If we try to printk() we'll probably make matters worse, by trying
1926 * to retake locks this cpu already holds or causing more errors. So
1927 * just bump a counter, and we'll report these counter bumps above.
1928 */
1929 void sun4v_nonresum_overflow(struct pt_regs *regs)
1931 /* XXX Actually even this can make not that much sense. Perhaps
1932 * XXX we should just pull the plug and panic directly from here?
1933 */
1934 atomic_inc(&sun4v_nonresum_oflow_cnt);
1937 unsigned long sun4v_err_itlb_vaddr;
1938 unsigned long sun4v_err_itlb_ctx;
1939 unsigned long sun4v_err_itlb_pte;
1940 unsigned long sun4v_err_itlb_error;
1942 void sun4v_itlb_error_report(struct pt_regs *regs, int tl)
1944 if (tl > 1)
1945 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
1947 printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n",
1948 regs->tpc, tl);
1949 printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] "
1950 "pte[%lx] error[%lx]\n",
1951 sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx,
1952 sun4v_err_itlb_pte, sun4v_err_itlb_error);
1954 prom_halt();
1957 unsigned long sun4v_err_dtlb_vaddr;
1958 unsigned long sun4v_err_dtlb_ctx;
1959 unsigned long sun4v_err_dtlb_pte;
1960 unsigned long sun4v_err_dtlb_error;
1962 void sun4v_dtlb_error_report(struct pt_regs *regs, int tl)
1964 if (tl > 1)
1965 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
1967 printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n",
1968 regs->tpc, tl);
1969 printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] "
1970 "pte[%lx] error[%lx]\n",
1971 sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx,
1972 sun4v_err_dtlb_pte, sun4v_err_dtlb_error);
1974 prom_halt();
1977 void hypervisor_tlbop_error(unsigned long err, unsigned long op)
1979 printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n",
1980 err, op);
1983 void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op)
1985 printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n",
1986 err, op);
1989 void do_fpe_common(struct pt_regs *regs)
1991 if (regs->tstate & TSTATE_PRIV) {
1992 regs->tpc = regs->tnpc;
1993 regs->tnpc += 4;
1994 } else {
1995 unsigned long fsr = current_thread_info()->xfsr[0];
1996 siginfo_t info;
1998 if (test_thread_flag(TIF_32BIT)) {
1999 regs->tpc &= 0xffffffff;
2000 regs->tnpc &= 0xffffffff;
2002 info.si_signo = SIGFPE;
2003 info.si_errno = 0;
2004 info.si_addr = (void __user *)regs->tpc;
2005 info.si_trapno = 0;
2006 info.si_code = __SI_FAULT;
2007 if ((fsr & 0x1c000) == (1 << 14)) {
2008 if (fsr & 0x10)
2009 info.si_code = FPE_FLTINV;
2010 else if (fsr & 0x08)
2011 info.si_code = FPE_FLTOVF;
2012 else if (fsr & 0x04)
2013 info.si_code = FPE_FLTUND;
2014 else if (fsr & 0x02)
2015 info.si_code = FPE_FLTDIV;
2016 else if (fsr & 0x01)
2017 info.si_code = FPE_FLTRES;
2019 force_sig_info(SIGFPE, &info, current);
2023 void do_fpieee(struct pt_regs *regs)
2025 if (notify_die(DIE_TRAP, "fpu exception ieee", regs,
2026 0, 0x24, SIGFPE) == NOTIFY_STOP)
2027 return;
2029 do_fpe_common(regs);
2032 extern int do_mathemu(struct pt_regs *, struct fpustate *);
2034 void do_fpother(struct pt_regs *regs)
2036 struct fpustate *f = FPUSTATE;
2037 int ret = 0;
2039 if (notify_die(DIE_TRAP, "fpu exception other", regs,
2040 0, 0x25, SIGFPE) == NOTIFY_STOP)
2041 return;
2043 switch ((current_thread_info()->xfsr[0] & 0x1c000)) {
2044 case (2 << 14): /* unfinished_FPop */
2045 case (3 << 14): /* unimplemented_FPop */
2046 ret = do_mathemu(regs, f);
2047 break;
2049 if (ret)
2050 return;
2051 do_fpe_common(regs);
2054 void do_tof(struct pt_regs *regs)
2056 siginfo_t info;
2058 if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs,
2059 0, 0x26, SIGEMT) == NOTIFY_STOP)
2060 return;
2062 if (regs->tstate & TSTATE_PRIV)
2063 die_if_kernel("Penguin overflow trap from kernel mode", regs);
2064 if (test_thread_flag(TIF_32BIT)) {
2065 regs->tpc &= 0xffffffff;
2066 regs->tnpc &= 0xffffffff;
2068 info.si_signo = SIGEMT;
2069 info.si_errno = 0;
2070 info.si_code = EMT_TAGOVF;
2071 info.si_addr = (void __user *)regs->tpc;
2072 info.si_trapno = 0;
2073 force_sig_info(SIGEMT, &info, current);
2076 void do_div0(struct pt_regs *regs)
2078 siginfo_t info;
2080 if (notify_die(DIE_TRAP, "integer division by zero", regs,
2081 0, 0x28, SIGFPE) == NOTIFY_STOP)
2082 return;
2084 if (regs->tstate & TSTATE_PRIV)
2085 die_if_kernel("TL0: Kernel divide by zero.", regs);
2086 if (test_thread_flag(TIF_32BIT)) {
2087 regs->tpc &= 0xffffffff;
2088 regs->tnpc &= 0xffffffff;
2090 info.si_signo = SIGFPE;
2091 info.si_errno = 0;
2092 info.si_code = FPE_INTDIV;
2093 info.si_addr = (void __user *)regs->tpc;
2094 info.si_trapno = 0;
2095 force_sig_info(SIGFPE, &info, current);
2098 void instruction_dump (unsigned int *pc)
2100 int i;
2102 if ((((unsigned long) pc) & 3))
2103 return;
2105 printk("Instruction DUMP:");
2106 for (i = -3; i < 6; i++)
2107 printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>');
2108 printk("\n");
2111 static void user_instruction_dump (unsigned int __user *pc)
2113 int i;
2114 unsigned int buf[9];
2116 if ((((unsigned long) pc) & 3))
2117 return;
2119 if (copy_from_user(buf, pc - 3, sizeof(buf)))
2120 return;
2122 printk("Instruction DUMP:");
2123 for (i = 0; i < 9; i++)
2124 printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>');
2125 printk("\n");
2128 void show_stack(struct task_struct *tsk, unsigned long *_ksp)
2130 unsigned long pc, fp, thread_base, ksp;
2131 void *tp = task_stack_page(tsk);
2132 struct reg_window *rw;
2133 int count = 0;
2135 ksp = (unsigned long) _ksp;
2137 if (tp == current_thread_info())
2138 flushw_all();
2140 fp = ksp + STACK_BIAS;
2141 thread_base = (unsigned long) tp;
2143 printk("Call Trace:");
2144 #ifdef CONFIG_KALLSYMS
2145 printk("\n");
2146 #endif
2147 do {
2148 /* Bogus frame pointer? */
2149 if (fp < (thread_base + sizeof(struct thread_info)) ||
2150 fp >= (thread_base + THREAD_SIZE))
2151 break;
2152 rw = (struct reg_window *)fp;
2153 pc = rw->ins[7];
2154 printk(" [%016lx] ", pc);
2155 print_symbol("%s\n", pc);
2156 fp = rw->ins[6] + STACK_BIAS;
2157 } while (++count < 16);
2158 #ifndef CONFIG_KALLSYMS
2159 printk("\n");
2160 #endif
2163 void dump_stack(void)
2165 unsigned long *ksp;
2167 __asm__ __volatile__("mov %%fp, %0"
2168 : "=r" (ksp));
2169 show_stack(current, ksp);
2172 EXPORT_SYMBOL(dump_stack);
2174 static inline int is_kernel_stack(struct task_struct *task,
2175 struct reg_window *rw)
2177 unsigned long rw_addr = (unsigned long) rw;
2178 unsigned long thread_base, thread_end;
2180 if (rw_addr < PAGE_OFFSET) {
2181 if (task != &init_task)
2182 return 0;
2185 thread_base = (unsigned long) task_stack_page(task);
2186 thread_end = thread_base + sizeof(union thread_union);
2187 if (rw_addr >= thread_base &&
2188 rw_addr < thread_end &&
2189 !(rw_addr & 0x7UL))
2190 return 1;
2192 return 0;
2195 static inline struct reg_window *kernel_stack_up(struct reg_window *rw)
2197 unsigned long fp = rw->ins[6];
2199 if (!fp)
2200 return NULL;
2202 return (struct reg_window *) (fp + STACK_BIAS);
2205 void die_if_kernel(char *str, struct pt_regs *regs)
2207 static int die_counter;
2208 extern void smp_report_regs(void);
2209 int count = 0;
2211 /* Amuse the user. */
2212 printk(
2213 " \\|/ ____ \\|/\n"
2214 " \"@'/ .. \\`@\"\n"
2215 " /_| \\__/ |_\\\n"
2216 " \\__U_/\n");
2218 printk("%s(%d): %s [#%d]\n", current->comm, current->pid, str, ++die_counter);
2219 notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV);
2220 __asm__ __volatile__("flushw");
2221 __show_regs(regs);
2222 if (regs->tstate & TSTATE_PRIV) {
2223 struct reg_window *rw = (struct reg_window *)
2224 (regs->u_regs[UREG_FP] + STACK_BIAS);
2226 /* Stop the back trace when we hit userland or we
2227 * find some badly aligned kernel stack.
2228 */
2229 while (rw &&
2230 count++ < 30&&
2231 is_kernel_stack(current, rw)) {
2232 printk("Caller[%016lx]", rw->ins[7]);
2233 print_symbol(": %s", rw->ins[7]);
2234 printk("\n");
2236 rw = kernel_stack_up(rw);
2238 instruction_dump ((unsigned int *) regs->tpc);
2239 } else {
2240 if (test_thread_flag(TIF_32BIT)) {
2241 regs->tpc &= 0xffffffff;
2242 regs->tnpc &= 0xffffffff;
2244 user_instruction_dump ((unsigned int __user *) regs->tpc);
2246 #if 0
2247 #ifdef CONFIG_SMP
2248 smp_report_regs();
2249 #endif
2250 #endif
2251 if (regs->tstate & TSTATE_PRIV)
2252 do_exit(SIGKILL);
2253 do_exit(SIGSEGV);
2256 extern int handle_popc(u32 insn, struct pt_regs *regs);
2257 extern int handle_ldf_stq(u32 insn, struct pt_regs *regs);
2259 void do_illegal_instruction(struct pt_regs *regs)
2261 unsigned long pc = regs->tpc;
2262 unsigned long tstate = regs->tstate;
2263 u32 insn;
2264 siginfo_t info;
2266 if (notify_die(DIE_TRAP, "illegal instruction", regs,
2267 0, 0x10, SIGILL) == NOTIFY_STOP)
2268 return;
2270 if (tstate & TSTATE_PRIV)
2271 die_if_kernel("Kernel illegal instruction", regs);
2272 if (test_thread_flag(TIF_32BIT))
2273 pc = (u32)pc;
2274 if (get_user(insn, (u32 __user *) pc) != -EFAULT) {
2275 if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ {
2276 if (handle_popc(insn, regs))
2277 return;
2278 } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ {
2279 if (handle_ldf_stq(insn, regs))
2280 return;
2281 } else if (tlb_type == hypervisor) {
2282 extern int vis_emul(struct pt_regs *, unsigned int);
2284 if (!vis_emul(regs, insn))
2285 return;
2288 info.si_signo = SIGILL;
2289 info.si_errno = 0;
2290 info.si_code = ILL_ILLOPC;
2291 info.si_addr = (void __user *)pc;
2292 info.si_trapno = 0;
2293 force_sig_info(SIGILL, &info, current);
2296 extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn);
2298 void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr)
2300 siginfo_t info;
2302 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2303 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2304 return;
2306 if (regs->tstate & TSTATE_PRIV) {
2307 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2308 return;
2310 info.si_signo = SIGBUS;
2311 info.si_errno = 0;
2312 info.si_code = BUS_ADRALN;
2313 info.si_addr = (void __user *)sfar;
2314 info.si_trapno = 0;
2315 force_sig_info(SIGBUS, &info, current);
2318 void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx)
2320 siginfo_t info;
2322 if (notify_die(DIE_TRAP, "memory address unaligned", regs,
2323 0, 0x34, SIGSEGV) == NOTIFY_STOP)
2324 return;
2326 if (regs->tstate & TSTATE_PRIV) {
2327 kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc));
2328 return;
2330 info.si_signo = SIGBUS;
2331 info.si_errno = 0;
2332 info.si_code = BUS_ADRALN;
2333 info.si_addr = (void __user *) addr;
2334 info.si_trapno = 0;
2335 force_sig_info(SIGBUS, &info, current);
2338 void do_privop(struct pt_regs *regs)
2340 siginfo_t info;
2342 if (notify_die(DIE_TRAP, "privileged operation", regs,
2343 0, 0x11, SIGILL) == NOTIFY_STOP)
2344 return;
2346 if (test_thread_flag(TIF_32BIT)) {
2347 regs->tpc &= 0xffffffff;
2348 regs->tnpc &= 0xffffffff;
2350 info.si_signo = SIGILL;
2351 info.si_errno = 0;
2352 info.si_code = ILL_PRVOPC;
2353 info.si_addr = (void __user *)regs->tpc;
2354 info.si_trapno = 0;
2355 force_sig_info(SIGILL, &info, current);
2358 void do_privact(struct pt_regs *regs)
2360 do_privop(regs);
2363 /* Trap level 1 stuff or other traps we should never see... */
2364 void do_cee(struct pt_regs *regs)
2366 die_if_kernel("TL0: Cache Error Exception", regs);
2369 void do_cee_tl1(struct pt_regs *regs)
2371 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2372 die_if_kernel("TL1: Cache Error Exception", regs);
2375 void do_dae_tl1(struct pt_regs *regs)
2377 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2378 die_if_kernel("TL1: Data Access Exception", regs);
2381 void do_iae_tl1(struct pt_regs *regs)
2383 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2384 die_if_kernel("TL1: Instruction Access Exception", regs);
2387 void do_div0_tl1(struct pt_regs *regs)
2389 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2390 die_if_kernel("TL1: DIV0 Exception", regs);
2393 void do_fpdis_tl1(struct pt_regs *regs)
2395 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2396 die_if_kernel("TL1: FPU Disabled", regs);
2399 void do_fpieee_tl1(struct pt_regs *regs)
2401 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2402 die_if_kernel("TL1: FPU IEEE Exception", regs);
2405 void do_fpother_tl1(struct pt_regs *regs)
2407 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2408 die_if_kernel("TL1: FPU Other Exception", regs);
2411 void do_ill_tl1(struct pt_regs *regs)
2413 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2414 die_if_kernel("TL1: Illegal Instruction Exception", regs);
2417 void do_irq_tl1(struct pt_regs *regs)
2419 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2420 die_if_kernel("TL1: IRQ Exception", regs);
2423 void do_lddfmna_tl1(struct pt_regs *regs)
2425 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2426 die_if_kernel("TL1: LDDF Exception", regs);
2429 void do_stdfmna_tl1(struct pt_regs *regs)
2431 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2432 die_if_kernel("TL1: STDF Exception", regs);
2435 void do_paw(struct pt_regs *regs)
2437 die_if_kernel("TL0: Phys Watchpoint Exception", regs);
2440 void do_paw_tl1(struct pt_regs *regs)
2442 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2443 die_if_kernel("TL1: Phys Watchpoint Exception", regs);
2446 void do_vaw(struct pt_regs *regs)
2448 die_if_kernel("TL0: Virt Watchpoint Exception", regs);
2451 void do_vaw_tl1(struct pt_regs *regs)
2453 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2454 die_if_kernel("TL1: Virt Watchpoint Exception", regs);
2457 void do_tof_tl1(struct pt_regs *regs)
2459 dump_tl1_traplog((struct tl1_traplog *)(regs + 1));
2460 die_if_kernel("TL1: Tag Overflow Exception", regs);
2463 void do_getpsr(struct pt_regs *regs)
2465 regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate);
2466 regs->tpc = regs->tnpc;
2467 regs->tnpc += 4;
2468 if (test_thread_flag(TIF_32BIT)) {
2469 regs->tpc &= 0xffffffff;
2470 regs->tnpc &= 0xffffffff;
2474 struct trap_per_cpu trap_block[NR_CPUS];
2476 /* This can get invoked before sched_init() so play it super safe
2477 * and use hard_smp_processor_id().
2478 */
2479 void init_cur_cpu_trap(struct thread_info *t)
2481 int cpu = hard_smp_processor_id();
2482 struct trap_per_cpu *p = &trap_block[cpu];
2484 p->thread = t;
2485 p->pgd_paddr = 0;
2488 extern void thread_info_offsets_are_bolixed_dave(void);
2489 extern void trap_per_cpu_offsets_are_bolixed_dave(void);
2490 extern void tsb_config_offsets_are_bolixed_dave(void);
2492 /* Only invoked on boot processor. */
2493 void __init trap_init(void)
2495 /* Compile time sanity check. */
2496 if (TI_TASK != offsetof(struct thread_info, task) ||
2497 TI_FLAGS != offsetof(struct thread_info, flags) ||
2498 TI_CPU != offsetof(struct thread_info, cpu) ||
2499 TI_FPSAVED != offsetof(struct thread_info, fpsaved) ||
2500 TI_KSP != offsetof(struct thread_info, ksp) ||
2501 TI_FAULT_ADDR != offsetof(struct thread_info, fault_address) ||
2502 TI_KREGS != offsetof(struct thread_info, kregs) ||
2503 TI_UTRAPS != offsetof(struct thread_info, utraps) ||
2504 TI_EXEC_DOMAIN != offsetof(struct thread_info, exec_domain) ||
2505 TI_REG_WINDOW != offsetof(struct thread_info, reg_window) ||
2506 TI_RWIN_SPTRS != offsetof(struct thread_info, rwbuf_stkptrs) ||
2507 TI_GSR != offsetof(struct thread_info, gsr) ||
2508 TI_XFSR != offsetof(struct thread_info, xfsr) ||
2509 TI_USER_CNTD0 != offsetof(struct thread_info, user_cntd0) ||
2510 TI_USER_CNTD1 != offsetof(struct thread_info, user_cntd1) ||
2511 TI_KERN_CNTD0 != offsetof(struct thread_info, kernel_cntd0) ||
2512 TI_KERN_CNTD1 != offsetof(struct thread_info, kernel_cntd1) ||
2513 TI_PCR != offsetof(struct thread_info, pcr_reg) ||
2514 TI_PRE_COUNT != offsetof(struct thread_info, preempt_count) ||
2515 TI_NEW_CHILD != offsetof(struct thread_info, new_child) ||
2516 TI_SYS_NOERROR != offsetof(struct thread_info, syscall_noerror) ||
2517 TI_RESTART_BLOCK != offsetof(struct thread_info, restart_block) ||
2518 TI_KUNA_REGS != offsetof(struct thread_info, kern_una_regs) ||
2519 TI_KUNA_INSN != offsetof(struct thread_info, kern_una_insn) ||
2520 TI_FPREGS != offsetof(struct thread_info, fpregs) ||
2521 (TI_FPREGS & (64 - 1)))
2522 thread_info_offsets_are_bolixed_dave();
2524 if (TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu, thread) ||
2525 (TRAP_PER_CPU_PGD_PADDR !=
2526 offsetof(struct trap_per_cpu, pgd_paddr)) ||
2527 (TRAP_PER_CPU_CPU_MONDO_PA !=
2528 offsetof(struct trap_per_cpu, cpu_mondo_pa)) ||
2529 (TRAP_PER_CPU_DEV_MONDO_PA !=
2530 offsetof(struct trap_per_cpu, dev_mondo_pa)) ||
2531 (TRAP_PER_CPU_RESUM_MONDO_PA !=
2532 offsetof(struct trap_per_cpu, resum_mondo_pa)) ||
2533 (TRAP_PER_CPU_RESUM_KBUF_PA !=
2534 offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) ||
2535 (TRAP_PER_CPU_NONRESUM_MONDO_PA !=
2536 offsetof(struct trap_per_cpu, nonresum_mondo_pa)) ||
2537 (TRAP_PER_CPU_NONRESUM_KBUF_PA !=
2538 offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) ||
2539 (TRAP_PER_CPU_FAULT_INFO !=
2540 offsetof(struct trap_per_cpu, fault_info)) ||
2541 (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA !=
2542 offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) ||
2543 (TRAP_PER_CPU_CPU_LIST_PA !=
2544 offsetof(struct trap_per_cpu, cpu_list_pa)) ||
2545 (TRAP_PER_CPU_TSB_HUGE !=
2546 offsetof(struct trap_per_cpu, tsb_huge)) ||
2547 (TRAP_PER_CPU_TSB_HUGE_TEMP !=
2548 offsetof(struct trap_per_cpu, tsb_huge_temp)) ||
2549 (TRAP_PER_CPU_IRQ_WORKLIST !=
2550 offsetof(struct trap_per_cpu, irq_worklist)))
2551 trap_per_cpu_offsets_are_bolixed_dave();
2553 if ((TSB_CONFIG_TSB !=
2554 offsetof(struct tsb_config, tsb)) ||
2555 (TSB_CONFIG_RSS_LIMIT !=
2556 offsetof(struct tsb_config, tsb_rss_limit)) ||
2557 (TSB_CONFIG_NENTRIES !=
2558 offsetof(struct tsb_config, tsb_nentries)) ||
2559 (TSB_CONFIG_REG_VAL !=
2560 offsetof(struct tsb_config, tsb_reg_val)) ||
2561 (TSB_CONFIG_MAP_VADDR !=
2562 offsetof(struct tsb_config, tsb_map_vaddr)) ||
2563 (TSB_CONFIG_MAP_PTE !=
2564 offsetof(struct tsb_config, tsb_map_pte)))
2565 tsb_config_offsets_are_bolixed_dave();
2567 /* Attach to the address space of init_task. On SMP we
2568 * do this in smp.c:smp_callin for other cpus.
2569 */
2570 atomic_inc(&init_mm.mm_count);
2571 current->active_mm = &init_mm;