Do not execute exception handlers with disabled interrupts.

We should not call vm_fault(), or send a signal, with interrupts
disabled.  MI kernel code is not prepared for such environment, not to
mention that this increases system latency, since code appears to be
executing as being under spinlock.

The FAR register for data aborts is read before the interrupts are
enabled, to avoid its corruption due to nested exception or context
switch.

Add asserts, similar to the checks done by other architectures, about
not taking page faults in non-sleepable contexts, rather than die with
late and somewhat confusing witness diagnostic.

Reviewed by:	andrew
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D3669

diff --git a/sys/arm64/arm64/trap.c b/sys/arm64/arm64/trap.c
index 394d6b41a39c38c4dd237ea3b302b42186676c0d..e54c20e54ce90090a9d5cd4d6e3ab3980dcc5c89 100644 (file)
--- a/sys/arm64/arm64/trap.c
+++ b/sys/arm64/arm64/trap.c
@@ -76,6 +76,7 @@ extern register_t fsu_intr_fault;
 void do_el1h_sync(struct trapframe *);
 void do_el0_sync(struct trapframe *);
 void do_el0_error(struct trapframe *);
+static void print_registers(struct trapframe *frame);
 
 int (*dtrace_invop_jump_addr)(struct trapframe *);
 
@@ -144,7 +145,7 @@ svc_handler(struct trapframe *frame)
 }
 
 static void
-data_abort(struct trapframe *frame, uint64_t esr, int lower)
+data_abort(struct trapframe *frame, uint64_t esr, uint64_t far, int lower)
 {
        struct vm_map *map;
        struct thread *td;
@@ -152,7 +153,6 @@ data_abort(struct trapframe *frame, uint64_t esr, int lower)
        struct pcb *pcb;
        vm_prot_t ftype;
        vm_offset_t va;
-       uint64_t far;
        int error, sig, ucode;
 
        /*
@@ -181,17 +181,23 @@ data_abort(struct trapframe *frame, uint64_t esr, int lower)
                return;
        }
 
-       far = READ_SPECIALREG(far_el1);
-       p = td->td_proc;
+       KASSERT(td->td_md.md_spinlock_count == 0,
+           ("data abort with spinlock held"));
+       if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
+           WARN_GIANTOK, NULL, "Kernel page fault") != 0) {
+               print_registers(frame);
+               panic("data abort in critical section or under mutex");
+       }
 
+       p = td->td_proc;
        if (lower)
-               map = &td->td_proc->p_vmspace->vm_map;
+               map = &p->p_vmspace->vm_map;
        else {
                /* The top bit tells us which range to use */
                if ((far >> 63) == 1)
                        map = kernel_map;
                else
-                       map = &td->td_proc->p_vmspace->vm_map;
+                       map = &p->p_vmspace->vm_map;
        }
 
        va = trunc_page(far);
@@ -246,7 +252,7 @@ void
 do_el1h_sync(struct trapframe *frame)
 {
        uint32_t exception;
-       uint64_t esr;
+       uint64_t esr, far;
 
        /* Read the esr register to get the exception details */
        esr = READ_SPECIALREG(esr_el1);
@@ -281,7 +287,9 @@ do_el1h_sync(struct trapframe *frame)
                print_registers(frame);
                panic("VFP exception in the kernel");
        case EXCP_DATA_ABORT:
-               data_abort(frame, esr, 0);
+               far = READ_SPECIALREG(far_el1);
+               intr_enable();
+               data_abort(frame, esr, far, 0);
                break;
        case EXCP_BRK:
 #ifdef KDTRACE_HOOKS
@@ -328,7 +336,7 @@ do_el0_sync(struct trapframe *frame)
 {
        struct thread *td;
        uint32_t exception;
-       uint64_t esr;
+       uint64_t esr, far;
 
        /* Check we have a sane environment when entering from userland */
        KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
@@ -337,6 +345,13 @@ do_el0_sync(struct trapframe *frame)
 
        esr = READ_SPECIALREG(esr_el1);
        exception = ESR_ELx_EXCEPTION(esr);
+       switch (exception) {
+       case EXCP_INSN_ABORT_L:
+       case EXCP_DATA_ABORT_L:
+       case EXCP_DATA_ABORT:
+               far = READ_SPECIALREG(far_el1);
+       }
+       intr_enable();
 
        CTR4(KTR_TRAP,
            "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p",
@@ -352,18 +367,12 @@ do_el0_sync(struct trapframe *frame)
 #endif
                break;
        case EXCP_SVC:
-               /*
-                * Ensure the svc_handler is being run with interrupts enabled.
-                * They will be automatically restored when returning from
-                * exception handler.
-                */
-               intr_enable();
                svc_handler(frame);
                break;
        case EXCP_INSN_ABORT_L:
        case EXCP_DATA_ABORT_L:
        case EXCP_DATA_ABORT:
-               data_abort(frame, esr, 1);
+               data_abort(frame, esr, far, 1);
                break;
        case EXCP_UNKNOWN:
                el0_excp_unknown(frame);