Use per-CPU event channel upcalls if available

A recent patch to Xen introduced a new HVM op to set a per-vcpu event
channel upcall. This patch adds code to make use of the latched interrupts
allocated by the FDO code to enable per-vcpu upcalls and adds an extra
EvtchnBind operation to a new v2 EVTCHN interface so that events can be
steered to a specified CPU.

Signed-off-by: Paul Durrant <paul.durrant@citrix.com>

diff --git a/include/evtchn_interface.h b/include/evtchn_interface.h
index 5898e08c58376286e6e05a182a6238f07c51507b..4ab2b2851bbf43b891db699d2bc2dee6d11c1832 100644 (file)
--- a/include/evtchn_interface.h
+++ b/include/evtchn_interface.h
@@ -112,6 +112,20 @@ typedef PXENBUS_EVTCHN_CHANNEL
     ...
     );
 
+/*! \typedef XENBUS_EVTCHN_BIND
+    \brief Bind an event channel to a specific CPU
+
+    \param Interface The interface header
+    \param Channel The channel handle
+    \param Cpu The CPU that should handle events
+*/
+typedef NTSTATUS
+(*XENBUS_EVTCHN_BIND)(
+    IN  PINTERFACE              Interface,
+    IN  PXENBUS_EVTCHN_CHANNEL  Channel,
+    IN  ULONG                   Cpu
+    );
+
 /*! \typedef XENBUS_EVTCHN_UNMASK
     \brief Unmask an event channel
 
@@ -196,7 +210,24 @@ struct _XENBUS_EVTCHN_INTERFACE_V1 {
     XENBUS_EVTCHN_CLOSE     EvtchnClose;
 };
 
-typedef struct _XENBUS_EVTCHN_INTERFACE_V1 XENBUS_EVTCHN_INTERFACE, *PXENBUS_EVTCHN_INTERFACE;
+/*! \struct _XENBUS_EVTCHN_INTERFACE_V2
+    \brief EVTCHN interface version 2
+    \ingroup interfaces
+*/
+struct _XENBUS_EVTCHN_INTERFACE_V2 {
+    INTERFACE               Interface;
+    XENBUS_EVTCHN_ACQUIRE   EvtchnAcquire;
+    XENBUS_EVTCHN_RELEASE   EvtchnRelease;
+    XENBUS_EVTCHN_OPEN      EvtchnOpen;
+    XENBUS_EVTCHN_BIND      EvtchnBind;
+    XENBUS_EVTCHN_UNMASK    EvtchnUnmask;
+    XENBUS_EVTCHN_SEND      EvtchnSend;
+    XENBUS_EVTCHN_TRIGGER   EvtchnTrigger;
+    XENBUS_EVTCHN_GET_PORT  EvtchnGetPort;
+    XENBUS_EVTCHN_CLOSE     EvtchnClose;
+};
+
+typedef struct _XENBUS_EVTCHN_INTERFACE_V2 XENBUS_EVTCHN_INTERFACE, *PXENBUS_EVTCHN_INTERFACE;
 
 /*! \def XENBUS_EVTCHN
     \brief Macro at assist in method invocation
@@ -207,7 +238,7 @@ typedef struct _XENBUS_EVTCHN_INTERFACE_V1 XENBUS_EVTCHN_INTERFACE, *PXENBUS_EVT
 #endif  // _WINDLL
 
 #define XENBUS_EVTCHN_INTERFACE_VERSION_MIN 1
-#define XENBUS_EVTCHN_INTERFACE_VERSION_MAX 1
+#define XENBUS_EVTCHN_INTERFACE_VERSION_MAX 2
 
 #endif  // _XENBUS_EVTCHN_INTERFACE_H
 

diff --git a/include/xen.h b/include/xen.h
index 0dabc74623598174959bfd449e5da47fb5993e08..84197a5b334e06beb25542f6dcef1d51eeade6dd 100644 (file)
--- a/include/xen.h
+++ b/include/xen.h
@@ -97,6 +97,14 @@ HvmPagetableDying(
     IN  PHYSICAL_ADDRESS    Address
     );
 
+__checkReturn
+XEN_API
+NTSTATUS
+HvmSetEvtchnUpcallVector(
+    IN  unsigned int    vcpu_id,
+    IN  UCHAR           Vector
+    );
+
 // MEMORY
 
 __checkReturn
@@ -196,6 +204,14 @@ EventChannelReset(
     VOID
     );
 
+__checkReturn
+XEN_API
+NTSTATUS
+EventChannelBindVirtualCpu(
+    IN  ULONG               LocalPort,
+    IN  unsigned int        vcpu_id
+    );
+
 // GRANT TABLE
 
 __checkReturn

diff --git a/src/xen/event_channel.c b/src/xen/event_channel.c
index aa87fd4b82977e179ea9277b98877de82c98cd52..94a6b82d944058482df95814f44fa7b19089240d 100644 (file)
--- a/src/xen/event_channel.c
+++ b/src/xen/event_channel.c
@@ -327,3 +327,33 @@ fail1:
 
     return status;
 }
+
+__checkReturn
+XEN_API
+NTSTATUS
+EventChannelBindVirtualCpu(
+    IN  ULONG               LocalPort,
+    IN  unsigned int        vcpu_id
+    )
+{
+    struct evtchn_bind_vcpu op;
+    LONG_PTR                rc;
+    NTSTATUS                status;
+
+    op.port = LocalPort;
+    op.vcpu = vcpu_id;
+
+    rc = EventChannelOp(EVTCHNOP_bind_vcpu, &op);
+
+    if (rc < 0) {
+        ERRNO_TO_STATUS(-rc, status);
+        goto fail1;
+    }
+
+    return STATUS_SUCCESS;
+
+fail1:
+    Error("fail1 (%08x)\n", status);
+
+    return status;
+}

diff --git a/src/xen/hvm.c b/src/xen/hvm.c
index 8135e97728db5034c8d2d507d543786a84feaf26..3e3e12cd55e712bfaa3223a0d075686fa289d718 100644 (file)
--- a/src/xen/hvm.c
+++ b/src/xen/hvm.c
@@ -166,3 +166,31 @@ HvmPagetableDying(
 fail1:
     return status;
 }
+
+__checkReturn
+XEN_API
+NTSTATUS
+HvmSetEvtchnUpcallVector(
+    IN  unsigned int                        vcpu_id,
+    IN  UCHAR                               Vector
+    )
+{
+    struct xen_hvm_set_evtchn_upcall_vector op;
+    LONG_PTR                                rc;
+    NTSTATUS                                status;
+
+    op.vcpu = vcpu_id;
+    op.vector = Vector;
+
+    rc = HvmOp(HVMOP_set_evtchn_upcall_vector, &op);
+
+    if (rc < 0) {
+        ERRNO_TO_STATUS(-rc, status);
+        goto fail1;
+    }
+
+    return STATUS_SUCCESS;
+
+fail1:
+    return status;
+}

diff --git a/src/xenbus/evtchn.c b/src/xenbus/evtchn.c
index 974dc2f4673b67c8ce91f1a216634666c3a7855c..9ebf0220ebf519b28ff87ae99acb63aabb8e94a1 100644 (file)
--- a/src/xenbus/evtchn.c
+++ b/src/xenbus/evtchn.c
@@ -81,13 +81,16 @@ struct _XENBUS_EVTCHN_CHANNEL {
     XENBUS_EVTCHN_PARAMETERS    Parameters;
     BOOLEAN                     Mask;
     ULONG                       LocalPort;
+    PXENBUS_INTERRUPT           Interrupt;
 };
 
 struct _XENBUS_EVTCHN_CONTEXT {
     PXENBUS_FDO                     Fdo;
     KSPIN_LOCK                      Lock;
     LONG                            References;
-    PXENBUS_INTERRUPT               Interrupt;
+    PXENBUS_INTERRUPT               LevelSensitiveInterrupt;
+    PXENBUS_INTERRUPT               LatchedInterrupt[MAXIMUM_PROCESSORS];
+    KAFFINITY                       Affinity;
     BOOLEAN                         Enabled;
     XENBUS_SUSPEND_INTERFACE        SuspendInterface;
     PXENBUS_SUSPEND_CALLBACK        SuspendCallbackEarly;
@@ -125,15 +128,39 @@ EvtchnInterruptEnable(
     IN  PXENBUS_EVTCHN_CONTEXT  Context
     )
 {
+    LONG                        Cpu;
     ULONG                       Line;
     NTSTATUS                    status;
 
-    Trace("<===>\n");
+    Trace("====>\n");
+
+    ASSERT3U(Context->Affinity, ==, 0);
+
+    Cpu = 0;
+    while (Cpu < KeNumberProcessors) {
+        unsigned int    vcpu_id;
+        UCHAR           Vector;
+
+        vcpu_id = SystemVirtualCpuIndex(Cpu);
+        Vector = FdoGetInterruptVector(Context->Fdo,
+                                       Context->LatchedInterrupt[Cpu]);
+
+        status = HvmSetEvtchnUpcallVector(vcpu_id, Vector);
+        if (NT_SUCCESS(status)) {
+            Info("CPU %u\n", Cpu);
+            Context->Affinity |= (KAFFINITY)1 << Cpu;
+        }
+
+        Cpu++;
+    }
 
-    Line = FdoGetInterruptLine(Context->Fdo, Context->Interrupt);
+    Line = FdoGetInterruptLine(Context->Fdo,
+                               Context->LevelSensitiveInterrupt);
 
     status = HvmSetParam(HVM_PARAM_CALLBACK_IRQ, Line);
     ASSERT(NT_SUCCESS(status));
+
+    Trace("<====\n");
 }
 
 static VOID
@@ -141,14 +168,29 @@ EvtchnInterruptDisable(
     IN  PXENBUS_EVTCHN_CONTEXT  Context
     )
 {
+    LONG                        Cpu;
     NTSTATUS                    status;
 
     UNREFERENCED_PARAMETER(Context);
 
-    Trace("<===>\n");
+    Trace("====>\n");
 
     status = HvmSetParam(HVM_PARAM_CALLBACK_IRQ, 0);
     ASSERT(NT_SUCCESS(status));
+
+    Cpu = KeNumberProcessors;
+    while (--Cpu >= 0) {
+        unsigned int    vcpu_id;
+
+        vcpu_id = SystemVirtualCpuIndex(Cpu);
+
+        (VOID) HvmSetEvtchnUpcallVector(vcpu_id, 0);
+        Context->Affinity &= ~((KAFFINITY)1 << Cpu);
+    }
+
+    ASSERT3U(Context->Affinity, ==, 0);
+
+    Trace("<====\n");
 }
 
 static FORCEINLINE
@@ -157,10 +199,11 @@ _IRQL_saves_
 _IRQL_raises_(HIGH_LEVEL)
 KIRQL
 __EvtchnAcquireInterruptLock(
-    IN  PXENBUS_EVTCHN_CONTEXT  Context
+    IN  PXENBUS_EVTCHN_CONTEXT  Context,
+    IN  PXENBUS_EVTCHN_CHANNEL  Channel
     )
 {
-    return FdoAcquireInterruptLock(Context->Fdo, Context->Interrupt);
+    return FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
 }
 
 static FORCEINLINE
@@ -168,10 +211,11 @@ __drv_requiresIRQL(HIGH_LEVEL)
 VOID
 __EvtchnReleaseInterruptLock(
     IN  PXENBUS_EVTCHN_CONTEXT      Context,
+    IN  PXENBUS_EVTCHN_CHANNEL      Channel,
     IN  __drv_restoresIRQL KIRQL    Irql
     )
 {
-    FdoReleaseInterruptLock(Context->Fdo, Context->Interrupt, Irql);
+    FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, Irql);
 }
 
 static NTSTATUS
@@ -356,6 +400,10 @@ EvtchnOpen(
 
     LocalPort = Channel->LocalPort;
 
+    Channel->Interrupt = (Context->Affinity != 0) ? // Latched available
+                         Context->LatchedInterrupt[0] :
+                         Context->LevelSensitiveInterrupt;
+
     status = XENBUS_EVTCHN_ABI(PortEnable,
                                &Context->EvtchnAbi,
                                LocalPort);
@@ -420,6 +468,89 @@ fail1:
     return NULL;
 }
 
+#define EVTCHN_SWAP_POINTER(_X, _Y)                             \
+        do {                                                    \
+            (_X) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y));  \
+            (_Y) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y));  \
+            (_X) = (PVOID)((ULONG_PTR)(_X) ^ (ULONG_PTR)(_Y));  \
+        } while (FALSE)
+
+static NTSTATUS
+EvtchnBind(
+    IN  PINTERFACE              Interface,
+    IN  PXENBUS_EVTCHN_CHANNEL  Channel,
+    IN  ULONG                   Cpu
+    )
+{
+    PXENBUS_EVTCHN_CONTEXT      Context = Interface->Context;
+    PXENBUS_INTERRUPT           Interrupt;
+    ULONG                       LocalPort;
+    unsigned int                vcpu_id;
+    KIRQL                       Irql;
+    NTSTATUS                    status;
+
+    status = STATUS_INVALID_PARAMETER;
+    if (Cpu >= (ULONG)KeNumberProcessors)
+        goto fail1;
+
+    ASSERT(Context->Enabled);
+
+    status = STATUS_NOT_SUPPORTED;
+    if (~Context->Affinity & ((KAFFINITY)1 << Cpu))
+        goto fail2;
+
+    Interrupt = Context->LatchedInterrupt[Cpu];
+
+    if (Channel->Interrupt == Interrupt)
+        goto done;
+
+    KeRaiseIrql(HIGH_LEVEL, &Irql);
+
+    // Make sure we always lock in a consistent order
+    if ((ULONG_PTR)Interrupt < (ULONG_PTR)Channel->Interrupt) {
+        (VOID) FdoAcquireInterruptLock(Context->Fdo, Interrupt);
+        (VOID) FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
+    } else {
+        (VOID) FdoAcquireInterruptLock(Context->Fdo, Channel->Interrupt);
+        (VOID) FdoAcquireInterruptLock(Context->Fdo, Interrupt);
+    }
+
+    LocalPort = Channel->LocalPort;
+    vcpu_id = SystemVirtualCpuIndex(Cpu);
+
+    status = EventChannelBindVirtualCpu(LocalPort, vcpu_id);
+    if (!NT_SUCCESS(status))
+        goto fail3;
+
+    EVTCHN_SWAP_POINTER(Channel->Interrupt, Interrupt);
+
+    FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, HIGH_LEVEL);
+    FdoReleaseInterruptLock(Context->Fdo, Interrupt, HIGH_LEVEL);
+
+    KeLowerIrql(Irql);
+
+    Info("[%u]: CPU %u\n", LocalPort, Cpu);
+
+done:
+    return STATUS_SUCCESS;
+
+fail3:
+    Error("fail3\n");
+
+    FdoReleaseInterruptLock(Context->Fdo, Channel->Interrupt, HIGH_LEVEL);
+    FdoReleaseInterruptLock(Context->Fdo, Interrupt, HIGH_LEVEL);
+
+    KeLowerIrql(Irql);
+
+fail2:
+    Error("fail2\n");
+
+fail1:
+    Error("fail1 (%08x)\n", status);
+
+    return status;
+}
+
 static BOOLEAN
 EvtchnUnmask(
     IN  PINTERFACE              Interface,
@@ -514,7 +645,7 @@ EvtchnTrigger(
 
     ASSERT3U(Channel->Magic, ==, XENBUS_EVTCHN_CHANNEL_MAGIC);
 
-    Irql = __EvtchnAcquireInterruptLock(Context);
+    Irql = __EvtchnAcquireInterruptLock(Context, Channel);
 
     ASSERT3U(KeGetCurrentIrql(), >=, DISPATCH_LEVEL);
 
@@ -525,7 +656,7 @@ EvtchnTrigger(
         DoneSomething = FALSE;
     }
 
-    __EvtchnReleaseInterruptLock(Context, Irql);
+    __EvtchnReleaseInterruptLock(Context, Channel, Irql);
 
     return DoneSomething;
 }
@@ -573,6 +704,8 @@ EvtchnClose(
         ASSERT(NT_SUCCESS(status));
     }
 
+    Channel->Interrupt = NULL;
+
     Channel->LocalPort = 0;
     RtlZeroMemory(&Channel->Parameters, sizeof (XENBUS_EVTCHN_PARAMETERS));
 
@@ -615,6 +748,8 @@ EvtchnPollCallback(
     BOOLEAN                 DoneSomething;
     NTSTATUS                status;
 
+    DoneSomething = FALSE;
+
     status = HashTableLookup(Context->Table,
                              LocalPort,
                              (PULONG_PTR)&Channel);
@@ -626,10 +761,20 @@ EvtchnPollCallback(
                           &Context->EvtchnAbi,
                           LocalPort);
 
-        DoneSomething = FALSE;
         goto done;
     }
 
+    if (Context->Affinity != 0) {
+        ULONG   Cpu;
+
+        ASSERT3U(KeGetCurrentIrql(), >=, DISPATCH_LEVEL);
+        Cpu = KeGetCurrentProcessorNumber();
+
+        // Only handle events on the correct CPU
+        if (Channel->Interrupt != Context->LatchedInterrupt[Cpu])
+            goto done;
+    }
+
     if (Channel->Mask)
         XENBUS_EVTCHN_ABI(PortMask,
                           &Context->EvtchnAbi,
@@ -757,8 +902,10 @@ EvtchnSuspendCallbackLate(
     status = EvtchnAbiAcquire(Context);
     ASSERT(NT_SUCCESS(status));
 
-    if (Context->Enabled)
+    if (Context->Enabled) {
+        EvtchnInterruptDisable(Context);
         EvtchnInterruptEnable(Context);
+    }
 }
 
 static VOID
@@ -852,6 +999,7 @@ EvtchnAcquire(
     PXENBUS_EVTCHN_CONTEXT  Context = Interface->Context;
     PXENBUS_FDO             Fdo = Context->Fdo;
     KIRQL                   Irql;
+    LONG                    Cpu;
     NTSTATUS                status;
 
     KeAcquireSpinLock(&Context->Lock, &Irql);
@@ -909,10 +1057,24 @@ EvtchnAcquire(
                                   0,
                                   EvtchnInterruptCallback,
                                   Context,
-                                  &Context->Interrupt);
+                                  &Context->LevelSensitiveInterrupt);
     if (!NT_SUCCESS(status))
         goto fail8;
 
+    Cpu = 0;
+    while (Cpu < KeNumberProcessors) {
+        status = FdoAllocateInterrupt(Fdo,
+                                      Latched,
+                                      Cpu,
+                                      EvtchnInterruptCallback,
+                                      Context,
+                                      &Context->LatchedInterrupt[Cpu]);
+        if (!NT_SUCCESS(status))
+            goto fail9;
+
+        Cpu++;
+    }
+
     Trace("<====\n");
 
 done:
@@ -920,6 +1082,17 @@ done:
 
     return STATUS_SUCCESS;
 
+fail9:
+    Error("fail9\n");
+
+    while (--Cpu >= 0) {
+        FdoFreeInterrupt(Fdo, Context->LatchedInterrupt[Cpu]);
+        Context->LatchedInterrupt[Cpu] = NULL;
+    }
+
+    FdoFreeInterrupt(Fdo, Context->LevelSensitiveInterrupt);
+    Context->LevelSensitiveInterrupt = NULL;
+
 fail8:
     Error("fail8\n");
 
@@ -980,7 +1153,9 @@ EvtchnRelease(
     )
 {
     PXENBUS_EVTCHN_CONTEXT  Context = Interface->Context;
+    PXENBUS_FDO             Fdo = Context->Fdo;
     KIRQL                   Irql;
+    LONG                    Cpu;
 
     KeAcquireSpinLock(&Context->Lock, &Irql);
 
@@ -992,8 +1167,14 @@ EvtchnRelease(
     if (!IsListEmpty(&Context->List))
         BUG("OUTSTANDING EVENT CHANNELS");
 
-    FdoFreeInterrupt(Context->Fdo, Context->Interrupt);
-    Context->Interrupt = NULL;
+    Cpu = KeNumberProcessors;
+    while (--Cpu >= 0) {
+        FdoFreeInterrupt(Fdo, Context->LatchedInterrupt[Cpu]);
+        Context->LatchedInterrupt[Cpu] = NULL;
+    }
+
+    FdoFreeInterrupt(Fdo, Context->LevelSensitiveInterrupt);
+    Context->LevelSensitiveInterrupt = NULL;
 
     EvtchnAbiRelease(Context);
 
@@ -1036,6 +1217,19 @@ static struct _XENBUS_EVTCHN_INTERFACE_V1 EvtchnInterfaceVersion1 = {
     EvtchnClose
 };
                      
+static struct _XENBUS_EVTCHN_INTERFACE_V2 EvtchnInterfaceVersion2 = {
+    { sizeof (struct _XENBUS_EVTCHN_INTERFACE_V2), 2, NULL, NULL, NULL },
+    EvtchnAcquire,
+    EvtchnRelease,
+    EvtchnOpen,
+    EvtchnBind,
+    EvtchnUnmask,
+    EvtchnSend,
+    EvtchnTrigger,
+    EvtchnGetPort,
+    EvtchnClose
+};
+
 NTSTATUS
 EvtchnInitialize(
     IN  PXENBUS_FDO             Fdo,
@@ -1149,6 +1343,23 @@ EvtchnGetInterface(
         status = STATUS_SUCCESS;
         break;
     }
+    case 2: {
+        struct _XENBUS_EVTCHN_INTERFACE_V2  *EvtchnInterface;
+
+        EvtchnInterface = (struct _XENBUS_EVTCHN_INTERFACE_V2 *)Interface;
+
+        status = STATUS_BUFFER_OVERFLOW;
+        if (Size < sizeof (struct _XENBUS_EVTCHN_INTERFACE_V2))
+            break;
+
+        *EvtchnInterface = EvtchnInterfaceVersion2;
+
+        ASSERT3U(Interface->Version, ==, Version);
+        Interface->Context = Context;
+
+        status = STATUS_SUCCESS;
+        break;
+    }
     default:
         status = STATUS_NOT_SUPPORTED;
         break;

diff --git a/src/xenbus/store.c b/src/xenbus/store.c
index 22c2a12a540502e6e0010baac8ea7cbb37469490..04fce3add510782eaf2553753d03224e1de1142c 100644 (file)
--- a/src/xenbus/store.c
+++ b/src/xenbus/store.c
@@ -1848,6 +1848,15 @@ StoreEnable(
                                      FALSE);
     ASSERT(Context->Channel != NULL);
 
+    // 
+    // Attempt to use some CPU other than 0 for events from
+    // xenstored.
+    //
+    (VOID) XENBUS_EVTCHN(Bind,
+                         &Context->EvtchnInterface,
+                         Context->Channel,
+                         KeNumberProcessors - 1);
+
     Pending = XENBUS_EVTCHN(Unmask,
                             &Context->EvtchnInterface,
                             Context->Channel,