ia64/xen-unstable

view linux-2.6-xen-sparse/arch/i386/kernel/entry-xen.S @ 9555:d76a7a40f3a9

Fix do_IRQ high bit masking.
Instead of setting the highest bit (which isn't easily done on native x86_64),
negate the interrupt vector stored in orig_{e,r}ax.
Also add patch for native build.

Signed-off-by: Christian Limpach <Christian.Limpach@cl.cam.ac.uk>
author cl349@firebug.cl.cam.ac.uk
date Fri Mar 31 17:44:26 2006 +0100 (2006-03-31)
parents ae6af19f47d3
children d73eeceeae69
line source
1 /*
2 * linux/arch/i386/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
7 /*
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
14 *
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
16 * on a 486.
17 *
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
23 *
24 * 0(%esp) - %ebx
25 * 4(%esp) - %ecx
26 * 8(%esp) - %edx
27 * C(%esp) - %esi
28 * 10(%esp) - %edi
29 * 14(%esp) - %ebp
30 * 18(%esp) - %eax
31 * 1C(%esp) - %ds
32 * 20(%esp) - %es
33 * 24(%esp) - orig_eax
34 * 28(%esp) - %eip
35 * 2C(%esp) - %cs
36 * 30(%esp) - %eflags
37 * 34(%esp) - %oldesp
38 * 38(%esp) - %oldss
39 *
40 * "current" is in register %ebx during any slow entries.
41 */
43 #include <linux/config.h>
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
48 #include <asm/smp.h>
49 #include <asm/page.h>
50 #include <asm/desc.h>
51 #include "irq_vectors.h"
52 #include <xen/interface/xen.h>
54 #define nr_syscalls ((syscall_table_size)/4)
56 EBX = 0x00
57 ECX = 0x04
58 EDX = 0x08
59 ESI = 0x0C
60 EDI = 0x10
61 EBP = 0x14
62 EAX = 0x18
63 DS = 0x1C
64 ES = 0x20
65 ORIG_EAX = 0x24
66 EIP = 0x28
67 CS = 0x2C
68 EFLAGS = 0x30
69 OLDESP = 0x34
70 OLDSS = 0x38
72 CF_MASK = 0x00000001
73 TF_MASK = 0x00000100
74 IF_MASK = 0x00000200
75 DF_MASK = 0x00000400
76 NT_MASK = 0x00004000
77 VM_MASK = 0x00020000
78 /* Pseudo-eflags. */
79 NMI_MASK = 0x80000000
81 #ifndef CONFIG_XEN
82 #define DISABLE_INTERRUPTS cli
83 #define ENABLE_INTERRUPTS sti
84 #else
85 /* Offsets into shared_info_t. */
86 #define evtchn_upcall_pending /* 0 */
87 #define evtchn_upcall_mask 1
89 #define sizeof_vcpu_shift 6
91 #ifdef CONFIG_SMP
92 #define GET_VCPU_INFO movl TI_cpu(%ebp),%esi ; \
93 shl $sizeof_vcpu_shift,%esi ; \
94 addl HYPERVISOR_shared_info,%esi
95 #else
96 #define GET_VCPU_INFO movl HYPERVISOR_shared_info,%esi
97 #endif
99 #define __DISABLE_INTERRUPTS movb $1,evtchn_upcall_mask(%esi)
100 #define __ENABLE_INTERRUPTS movb $0,evtchn_upcall_mask(%esi)
101 #define DISABLE_INTERRUPTS GET_VCPU_INFO ; \
102 __DISABLE_INTERRUPTS
103 #define ENABLE_INTERRUPTS GET_VCPU_INFO ; \
104 __ENABLE_INTERRUPTS
105 #define __TEST_PENDING testb $0xFF,evtchn_upcall_pending(%esi)
106 #endif
108 #ifdef CONFIG_PREEMPT
109 #define preempt_stop cli
110 #else
111 #define preempt_stop
112 #define resume_kernel restore_nocheck
113 #endif
115 #define SAVE_ALL \
116 cld; \
117 pushl %es; \
118 pushl %ds; \
119 pushl %eax; \
120 pushl %ebp; \
121 pushl %edi; \
122 pushl %esi; \
123 pushl %edx; \
124 pushl %ecx; \
125 pushl %ebx; \
126 movl $(__USER_DS), %edx; \
127 movl %edx, %ds; \
128 movl %edx, %es;
130 #define RESTORE_INT_REGS \
131 popl %ebx; \
132 popl %ecx; \
133 popl %edx; \
134 popl %esi; \
135 popl %edi; \
136 popl %ebp; \
137 popl %eax
139 #define RESTORE_REGS \
140 RESTORE_INT_REGS; \
141 1: popl %ds; \
142 2: popl %es; \
143 .section .fixup,"ax"; \
144 3: movl $0,(%esp); \
145 jmp 1b; \
146 4: movl $0,(%esp); \
147 jmp 2b; \
148 .previous; \
149 .section __ex_table,"a";\
150 .align 4; \
151 .long 1b,3b; \
152 .long 2b,4b; \
153 .previous
156 ENTRY(ret_from_fork)
157 pushl %eax
158 call schedule_tail
159 GET_THREAD_INFO(%ebp)
160 popl %eax
161 jmp syscall_exit
163 /*
164 * Return to user mode is not as complex as all this looks,
165 * but we want the default path for a system call return to
166 * go as quickly as possible which is why some of this is
167 * less clear than it otherwise should be.
168 */
170 # userspace resumption stub bypassing syscall exit tracing
171 ALIGN
172 ret_from_exception:
173 preempt_stop
174 ret_from_intr:
175 GET_THREAD_INFO(%ebp)
176 movl EFLAGS(%esp), %eax # mix EFLAGS and CS
177 movb CS(%esp), %al
178 testl $(VM_MASK | 2), %eax
179 jz resume_kernel
180 ENTRY(resume_userspace)
181 DISABLE_INTERRUPTS # make sure we don't miss an interrupt
182 # setting need_resched or sigpending
183 # between sampling and the iret
184 movl TI_flags(%ebp), %ecx
185 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
186 # int/exception return?
187 jne work_pending
188 jmp restore_all
190 #ifdef CONFIG_PREEMPT
191 ENTRY(resume_kernel)
192 cli
193 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
194 jnz restore_nocheck
195 need_resched:
196 movl TI_flags(%ebp), %ecx # need_resched set ?
197 testb $_TIF_NEED_RESCHED, %cl
198 jz restore_all
199 testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ?
200 jz restore_all
201 call preempt_schedule_irq
202 jmp need_resched
203 #endif
205 #ifdef CONFIG_X86_SYSENTER
206 /* SYSENTER_RETURN points to after the "sysenter" instruction in
207 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
209 # sysenter call handler stub
210 ENTRY(sysenter_entry)
211 movl TSS_sysenter_esp0(%esp),%esp
212 sysenter_past_esp:
213 sti
214 pushl $(__USER_DS)
215 pushl %ebp
216 pushfl
217 pushl $(__USER_CS)
218 pushl $SYSENTER_RETURN
220 /*
221 * Load the potential sixth argument from user stack.
222 * Careful about security.
223 */
224 cmpl $__PAGE_OFFSET-3,%ebp
225 jae syscall_fault
226 1: movl (%ebp),%ebp
227 .section __ex_table,"a"
228 .align 4
229 .long 1b,syscall_fault
230 .previous
232 pushl %eax
233 SAVE_ALL
234 GET_THREAD_INFO(%ebp)
236 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
237 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
238 jnz syscall_trace_entry
239 cmpl $(nr_syscalls), %eax
240 jae syscall_badsys
241 call *sys_call_table(,%eax,4)
242 movl %eax,EAX(%esp)
243 cli
244 movl TI_flags(%ebp), %ecx
245 testw $_TIF_ALLWORK_MASK, %cx
246 jne syscall_exit_work
247 /* if something modifies registers it must also disable sysexit */
248 movl EIP(%esp), %edx
249 movl OLDESP(%esp), %ecx
250 xorl %ebp,%ebp
251 sti
252 sysexit
253 #endif /* CONFIG_X86_SYSENTER */
256 # system call handler stub
257 ENTRY(system_call)
258 pushl %eax # save orig_eax
259 SAVE_ALL
260 GET_THREAD_INFO(%ebp)
261 # system call tracing in operation / emulation
262 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
263 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
264 jnz syscall_trace_entry
265 cmpl $(nr_syscalls), %eax
266 jae syscall_badsys
267 syscall_call:
268 call *sys_call_table(,%eax,4)
269 movl %eax,EAX(%esp) # store the return value
270 syscall_exit:
271 DISABLE_INTERRUPTS # make sure we don't miss an interrupt
272 # setting need_resched or sigpending
273 # between sampling and the iret
274 movl TI_flags(%ebp), %ecx
275 testw $_TIF_ALLWORK_MASK, %cx # current->work
276 jne syscall_exit_work
278 restore_all:
279 #ifndef CONFIG_XEN
280 movl EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
281 # Warning: OLDSS(%esp) contains the wrong/random values if we
282 # are returning to the kernel.
283 # See comments in process.c:copy_thread() for details.
284 movb OLDSS(%esp), %ah
285 movb CS(%esp), %al
286 andl $(VM_MASK | (4 << 8) | 3), %eax
287 cmpl $((4 << 8) | 3), %eax
288 je ldt_ss # returning to user-space with LDT SS
289 restore_nocheck:
290 #else
291 restore_nocheck:
292 movl EFLAGS(%esp), %eax
293 testl $(VM_MASK|NMI_MASK), %eax
294 jnz hypervisor_iret
295 shr $9, %eax # EAX[0] == IRET_EFLAGS.IF
296 GET_VCPU_INFO
297 andb evtchn_upcall_mask(%esi),%al
298 andb $1,%al # EAX[0] == IRET_EFLAGS.IF & event_mask
299 jnz restore_all_enable_events # != 0 => enable event delivery
300 #endif
301 RESTORE_REGS
302 addl $4, %esp
303 1: iret
304 .section .fixup,"ax"
305 iret_exc:
306 #ifndef CONFIG_XEN
307 sti
308 #endif
309 pushl $0 # no error code
310 pushl $do_iret_error
311 jmp error_code
312 .previous
313 .section __ex_table,"a"
314 .align 4
315 .long 1b,iret_exc
316 .previous
318 #ifndef CONFIG_XEN
319 ldt_ss:
320 larl OLDSS(%esp), %eax
321 jnz restore_nocheck
322 testl $0x00400000, %eax # returning to 32bit stack?
323 jnz restore_nocheck # allright, normal return
324 /* If returning to userspace with 16bit stack,
325 * try to fix the higher word of ESP, as the CPU
326 * won't restore it.
327 * This is an "official" bug of all the x86-compatible
328 * CPUs, which we can try to work around to make
329 * dosemu and wine happy. */
330 subl $8, %esp # reserve space for switch16 pointer
331 cli
332 movl %esp, %eax
333 /* Set up the 16bit stack frame with switch32 pointer on top,
334 * and a switch16 pointer on top of the current frame. */
335 call setup_x86_bogus_stack
336 RESTORE_REGS
337 lss 20+4(%esp), %esp # switch to 16bit stack
338 1: iret
339 .section __ex_table,"a"
340 .align 4
341 .long 1b,iret_exc
342 .previous
343 #else
344 hypervisor_iret:
345 andl $~NMI_MASK, EFLAGS(%esp)
346 RESTORE_REGS
347 addl $4, %esp
348 jmp hypercall_page + (__HYPERVISOR_iret * 32)
349 #endif
351 # perform work that needs to be done immediately before resumption
352 ALIGN
353 work_pending:
354 testb $_TIF_NEED_RESCHED, %cl
355 jz work_notifysig
356 work_resched:
357 call schedule
358 DISABLE_INTERRUPTS # make sure we don't miss an interrupt
359 # setting need_resched or sigpending
360 # between sampling and the iret
361 movl TI_flags(%ebp), %ecx
362 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
363 # than syscall tracing?
364 jz restore_all
365 testb $_TIF_NEED_RESCHED, %cl
366 jnz work_resched
368 work_notifysig: # deal with pending signals and
369 # notify-resume requests
370 testl $VM_MASK, EFLAGS(%esp)
371 movl %esp, %eax
372 jne work_notifysig_v86 # returning to kernel-space or
373 # vm86-space
374 xorl %edx, %edx
375 call do_notify_resume
376 jmp resume_userspace
378 ALIGN
379 work_notifysig_v86:
380 #ifdef CONFIG_VM86
381 pushl %ecx # save ti_flags for do_notify_resume
382 call save_v86_state # %eax contains pt_regs pointer
383 popl %ecx
384 movl %eax, %esp
385 xorl %edx, %edx
386 call do_notify_resume
387 jmp resume_userspace
388 #endif
390 # perform syscall exit tracing
391 ALIGN
392 syscall_trace_entry:
393 movl $-ENOSYS,EAX(%esp)
394 movl %esp, %eax
395 xorl %edx,%edx
396 call do_syscall_trace
397 cmpl $0, %eax
398 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
399 # so must skip actual syscall
400 movl ORIG_EAX(%esp), %eax
401 cmpl $(nr_syscalls), %eax
402 jnae syscall_call
403 jmp syscall_exit
405 # perform syscall exit tracing
406 ALIGN
407 syscall_exit_work:
408 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
409 jz work_pending
410 ENABLE_INTERRUPTS # could let do_syscall_trace() call
411 # schedule() instead
412 movl %esp, %eax
413 movl $1, %edx
414 call do_syscall_trace
415 jmp resume_userspace
417 ALIGN
418 syscall_fault:
419 pushl %eax # save orig_eax
420 SAVE_ALL
421 GET_THREAD_INFO(%ebp)
422 movl $-EFAULT,EAX(%esp)
423 jmp resume_userspace
425 ALIGN
426 syscall_badsys:
427 movl $-ENOSYS,EAX(%esp)
428 jmp resume_userspace
430 #ifndef CONFIG_XEN
431 #define FIXUP_ESPFIX_STACK \
432 movl %esp, %eax; \
433 /* switch to 32bit stack using the pointer on top of 16bit stack */ \
434 lss %ss:CPU_16BIT_STACK_SIZE-8, %esp; \
435 /* copy data from 16bit stack to 32bit stack */ \
436 call fixup_x86_bogus_stack; \
437 /* put ESP to the proper location */ \
438 movl %eax, %esp;
439 #define UNWIND_ESPFIX_STACK \
440 pushl %eax; \
441 movl %ss, %eax; \
442 /* see if on 16bit stack */ \
443 cmpw $__ESPFIX_SS, %ax; \
444 jne 28f; \
445 movl $__KERNEL_DS, %edx; \
446 movl %edx, %ds; \
447 movl %edx, %es; \
448 /* switch to 32bit stack */ \
449 FIXUP_ESPFIX_STACK \
450 28: popl %eax;
452 /*
453 * Build the entry stubs and pointer table with
454 * some assembler magic.
455 */
456 .data
457 ENTRY(interrupt)
458 .text
460 vector=0
461 ENTRY(irq_entries_start)
462 .rept NR_IRQS
463 ALIGN
464 1: pushl $~(vector)
465 jmp common_interrupt
466 .data
467 .long 1b
468 .text
469 vector=vector+1
470 .endr
472 ALIGN
473 common_interrupt:
474 SAVE_ALL
475 movl %esp,%eax
476 call do_IRQ
477 jmp ret_from_intr
479 #define BUILD_INTERRUPT(name, nr) \
480 ENTRY(name) \
481 pushl $~(nr); \
482 SAVE_ALL \
483 movl %esp,%eax; \
484 call smp_/**/name; \
485 jmp ret_from_intr;
487 /* The include is where all of the SMP etc. interrupts come from */
488 #include "entry_arch.h"
489 #else
490 #define UNWIND_ESPFIX_STACK
491 #endif
493 ENTRY(divide_error)
494 pushl $0 # no error code
495 pushl $do_divide_error
496 ALIGN
497 error_code:
498 pushl %ds
499 pushl %eax
500 xorl %eax, %eax
501 pushl %ebp
502 pushl %edi
503 pushl %esi
504 pushl %edx
505 decl %eax # eax = -1
506 pushl %ecx
507 pushl %ebx
508 cld
509 pushl %es
510 UNWIND_ESPFIX_STACK
511 popl %ecx
512 movl ES(%esp), %edi # get the function address
513 movl ORIG_EAX(%esp), %edx # get the error code
514 movl %eax, ORIG_EAX(%esp)
515 movl %ecx, ES(%esp)
516 movl $(__USER_DS), %ecx
517 movl %ecx, %ds
518 movl %ecx, %es
519 movl %esp,%eax # pt_regs pointer
520 call *%edi
521 jmp ret_from_exception
523 #ifdef CONFIG_XEN
524 # A note on the "critical region" in our callback handler.
525 # We want to avoid stacking callback handlers due to events occurring
526 # during handling of the last event. To do this, we keep events disabled
527 # until we've done all processing. HOWEVER, we must enable events before
528 # popping the stack frame (can't be done atomically) and so it would still
529 # be possible to get enough handler activations to overflow the stack.
530 # Although unlikely, bugs of that kind are hard to track down, so we'd
531 # like to avoid the possibility.
532 # So, on entry to the handler we detect whether we interrupted an
533 # existing activation in its critical region -- if so, we pop the current
534 # activation and restart the handler using the previous one.
535 ENTRY(hypervisor_callback)
536 pushl %eax
537 SAVE_ALL
538 movl EIP(%esp),%eax
539 cmpl $scrit,%eax
540 jb 11f
541 cmpl $ecrit,%eax
542 jb critical_region_fixup
543 11: push %esp
544 call evtchn_do_upcall
545 add $4,%esp
546 jmp ret_from_intr
548 ALIGN
549 restore_all_enable_events:
550 __ENABLE_INTERRUPTS
551 scrit: /**** START OF CRITICAL REGION ****/
552 __TEST_PENDING
553 jnz 14f # process more events if necessary...
554 RESTORE_REGS
555 addl $4, %esp
556 1: iret
557 .section __ex_table,"a"
558 .align 4
559 .long 1b,iret_exc
560 .previous
561 14: __DISABLE_INTERRUPTS
562 jmp 11b
563 ecrit: /**** END OF CRITICAL REGION ****/
564 # [How we do the fixup]. We want to merge the current stack frame with the
565 # just-interrupted frame. How we do this depends on where in the critical
566 # region the interrupted handler was executing, and so how many saved
567 # registers are in each frame. We do this quickly using the lookup table
568 # 'critical_fixup_table'. For each byte offset in the critical region, it
569 # provides the number of bytes which have already been popped from the
570 # interrupted stack frame.
571 critical_region_fixup:
572 addl $critical_fixup_table-scrit,%eax
573 movzbl (%eax),%eax # %eax contains num bytes popped
574 cmpb $0xff,%al # 0xff => vcpu_info critical region
575 jne 15f
576 GET_THREAD_INFO(%ebp)
577 xorl %eax,%eax
578 15: mov %esp,%esi
579 add %eax,%esi # %esi points at end of src region
580 mov %esp,%edi
581 add $0x34,%edi # %edi points at end of dst region
582 mov %eax,%ecx
583 shr $2,%ecx # convert words to bytes
584 je 17f # skip loop if nothing to copy
585 16: subl $4,%esi # pre-decrementing copy loop
586 subl $4,%edi
587 movl (%esi),%eax
588 movl %eax,(%edi)
589 loop 16b
590 17: movl %edi,%esp # final %edi is top of merged stack
591 jmp 11b
593 critical_fixup_table:
594 .byte 0xff,0xff,0xff # testb $0xff,(%esi) = __TEST_PENDING
595 .byte 0xff,0xff # jnz 14f
596 .byte 0x00 # pop %ebx
597 .byte 0x04 # pop %ecx
598 .byte 0x08 # pop %edx
599 .byte 0x0c # pop %esi
600 .byte 0x10 # pop %edi
601 .byte 0x14 # pop %ebp
602 .byte 0x18 # pop %eax
603 .byte 0x1c # pop %ds
604 .byte 0x20 # pop %es
605 .byte 0x24,0x24,0x24 # add $4,%esp
606 .byte 0x28 # iret
607 .byte 0xff,0xff,0xff,0xff # movb $1,1(%esi)
608 .byte 0x00,0x00 # jmp 11b
610 # Hypervisor uses this for application faults while it executes.
611 # We get here for two reasons:
612 # 1. Fault while reloading DS, ES, FS or GS
613 # 2. Fault while executing IRET
614 # Category 1 we fix up by reattempting the load, and zeroing the segment
615 # register if the load fails.
616 # Category 2 we fix up by jumping to do_iret_error. We cannot use the
617 # normal Linux return path in this case because if we use the IRET hypercall
618 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
619 # We distinguish between categories by maintaining a status value in EAX.
620 ENTRY(failsafe_callback)
621 pushl %eax
622 movl $1,%eax
623 1: mov 4(%esp),%ds
624 2: mov 8(%esp),%es
625 3: mov 12(%esp),%fs
626 4: mov 16(%esp),%gs
627 testl %eax,%eax
628 popl %eax
629 jz 5f
630 addl $16,%esp # EAX != 0 => Category 2 (Bad IRET)
631 jmp iret_exc
632 5: addl $16,%esp # EAX == 0 => Category 1 (Bad segment)
633 pushl $0
634 SAVE_ALL
635 jmp ret_from_exception
636 .section .fixup,"ax"; \
637 6: xorl %eax,%eax; \
638 movl %eax,4(%esp); \
639 jmp 1b; \
640 7: xorl %eax,%eax; \
641 movl %eax,8(%esp); \
642 jmp 2b; \
643 8: xorl %eax,%eax; \
644 movl %eax,12(%esp); \
645 jmp 3b; \
646 9: xorl %eax,%eax; \
647 movl %eax,16(%esp); \
648 jmp 4b; \
649 .previous; \
650 .section __ex_table,"a"; \
651 .align 4; \
652 .long 1b,6b; \
653 .long 2b,7b; \
654 .long 3b,8b; \
655 .long 4b,9b; \
656 .previous
657 #endif
659 ENTRY(coprocessor_error)
660 pushl $0
661 pushl $do_coprocessor_error
662 jmp error_code
664 ENTRY(simd_coprocessor_error)
665 pushl $0
666 pushl $do_simd_coprocessor_error
667 jmp error_code
669 ENTRY(device_not_available)
670 pushl $-1 # mark this as an int
671 SAVE_ALL
672 #ifndef CONFIG_XEN
673 movl %cr0, %eax
674 testl $0x4, %eax # EM (math emulation bit)
675 je device_available_emulate
676 pushl $0 # temporary storage for ORIG_EIP
677 call math_emulate
678 addl $4, %esp
679 jmp ret_from_exception
680 device_available_emulate:
681 #endif
682 preempt_stop
683 call math_state_restore
684 jmp ret_from_exception
686 #ifdef CONFIG_X86_SYSENTER
687 /*
688 * Debug traps and NMI can happen at the one SYSENTER instruction
689 * that sets up the real kernel stack. Check here, since we can't
690 * allow the wrong stack to be used.
691 *
692 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have
693 * already pushed 3 words if it hits on the sysenter instruction:
694 * eflags, cs and eip.
695 *
696 * We just load the right stack, and push the three (known) values
697 * by hand onto the new stack - while updating the return eip past
698 * the instruction that would have done it for sysenter.
699 */
700 #define FIX_STACK(offset, ok, label) \
701 cmpw $__KERNEL_CS,4(%esp); \
702 jne ok; \
703 label: \
704 movl TSS_sysenter_esp0+offset(%esp),%esp; \
705 pushfl; \
706 pushl $__KERNEL_CS; \
707 pushl $sysenter_past_esp
708 #endif /* CONFIG_X86_SYSENTER */
710 KPROBE_ENTRY(debug)
711 #ifdef CONFIG_X86_SYSENTER
712 cmpl $sysenter_entry,(%esp)
713 jne debug_stack_correct
714 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
715 debug_stack_correct:
716 #endif /* !CONFIG_X86_SYSENTER */
717 pushl $-1 # mark this as an int
718 SAVE_ALL
719 xorl %edx,%edx # error code 0
720 movl %esp,%eax # pt_regs pointer
721 call do_debug
722 jmp ret_from_exception
723 .previous .text
725 #ifndef CONFIG_XEN
726 /*
727 * NMI is doubly nasty. It can happen _while_ we're handling
728 * a debug fault, and the debug fault hasn't yet been able to
729 * clear up the stack. So we first check whether we got an
730 * NMI on the sysenter entry path, but after that we need to
731 * check whether we got an NMI on the debug path where the debug
732 * fault happened on the sysenter path.
733 */
734 ENTRY(nmi)
735 pushl %eax
736 movl %ss, %eax
737 cmpw $__ESPFIX_SS, %ax
738 popl %eax
739 je nmi_16bit_stack
740 cmpl $sysenter_entry,(%esp)
741 je nmi_stack_fixup
742 pushl %eax
743 movl %esp,%eax
744 /* Do not access memory above the end of our stack page,
745 * it might not exist.
746 */
747 andl $(THREAD_SIZE-1),%eax
748 cmpl $(THREAD_SIZE-20),%eax
749 popl %eax
750 jae nmi_stack_correct
751 cmpl $sysenter_entry,12(%esp)
752 je nmi_debug_stack_check
753 nmi_stack_correct:
754 pushl %eax
755 SAVE_ALL
756 xorl %edx,%edx # zero error code
757 movl %esp,%eax # pt_regs pointer
758 call do_nmi
759 jmp restore_all
761 nmi_stack_fixup:
762 FIX_STACK(12,nmi_stack_correct, 1)
763 jmp nmi_stack_correct
764 nmi_debug_stack_check:
765 cmpw $__KERNEL_CS,16(%esp)
766 jne nmi_stack_correct
767 cmpl $debug,(%esp)
768 jb nmi_stack_correct
769 cmpl $debug_esp_fix_insn,(%esp)
770 ja nmi_stack_correct
771 FIX_STACK(24,nmi_stack_correct, 1)
772 jmp nmi_stack_correct
774 nmi_16bit_stack:
775 /* create the pointer to lss back */
776 pushl %ss
777 pushl %esp
778 movzwl %sp, %esp
779 addw $4, (%esp)
780 /* copy the iret frame of 12 bytes */
781 .rept 3
782 pushl 16(%esp)
783 .endr
784 pushl %eax
785 SAVE_ALL
786 FIXUP_ESPFIX_STACK # %eax == %esp
787 xorl %edx,%edx # zero error code
788 call do_nmi
789 RESTORE_REGS
790 lss 12+4(%esp), %esp # back to 16bit stack
791 1: iret
792 .section __ex_table,"a"
793 .align 4
794 .long 1b,iret_exc
795 .previous
796 #else
797 ENTRY(nmi)
798 pushl %eax
799 SAVE_ALL
800 xorl %edx,%edx # zero error code
801 movl %esp,%eax # pt_regs pointer
802 call do_nmi
803 orl $NMI_MASK, EFLAGS(%esp)
804 jmp restore_all
805 #endif
807 KPROBE_ENTRY(int3)
808 pushl $-1 # mark this as an int
809 SAVE_ALL
810 xorl %edx,%edx # zero error code
811 movl %esp,%eax # pt_regs pointer
812 call do_int3
813 jmp ret_from_exception
814 .previous .text
816 ENTRY(overflow)
817 pushl $0
818 pushl $do_overflow
819 jmp error_code
821 ENTRY(bounds)
822 pushl $0
823 pushl $do_bounds
824 jmp error_code
826 ENTRY(invalid_op)
827 pushl $0
828 pushl $do_invalid_op
829 jmp error_code
831 ENTRY(coprocessor_segment_overrun)
832 pushl $0
833 pushl $do_coprocessor_segment_overrun
834 jmp error_code
836 ENTRY(invalid_TSS)
837 pushl $do_invalid_TSS
838 jmp error_code
840 ENTRY(segment_not_present)
841 pushl $do_segment_not_present
842 jmp error_code
844 ENTRY(stack_segment)
845 pushl $do_stack_segment
846 jmp error_code
848 KPROBE_ENTRY(general_protection)
849 pushl $do_general_protection
850 jmp error_code
851 .previous .text
853 ENTRY(alignment_check)
854 pushl $do_alignment_check
855 jmp error_code
857 KPROBE_ENTRY(page_fault)
858 pushl $do_page_fault
859 jmp error_code
860 .previous .text
862 #ifdef CONFIG_X86_MCE
863 ENTRY(machine_check)
864 pushl $0
865 pushl machine_check_vector
866 jmp error_code
867 #endif
869 ENTRY(fixup_4gb_segment)
870 pushl $do_fixup_4gb_segment
871 jmp error_code
873 .section .rodata,"a"
874 #include "syscall_table.S"
876 syscall_table_size=(.-sys_call_table)