re-work sync to do per-cpu callbacks

Signed-off-by: Paul Durrant <pdurrant@amazon.com>

diff --git a/src/xenbus/suspend.c b/src/xenbus/suspend.c
index 5667e8446176010f9d9a817493d2dd68f31f9358..77752b00d4904a10180fccae1150722364d8a427 100644 (file)
--- a/src/xenbus/suspend.c
+++ b/src/xenbus/suspend.c
@@ -51,6 +51,7 @@ struct _XENBUS_SUSPEND_CONTEXT {
     PXENBUS_FDO                 Fdo;
     KSPIN_LOCK                  Lock;
     LONG                        References;
+    BOOLEAN                     Success;
     ULONG                       Count;
     LIST_ENTRY                  EarlyList;
     LIST_ENTRY                  LateList;
@@ -180,6 +181,73 @@ __SuspendLogTimers(
               PerformanceFrequency.LowPart);
 }
 
+static VOID
+SuspendEarly(
+    IN  PVOID               Argument,
+    IN  ULONG               Cpu
+    )
+{
+    PXENBUS_SUSPEND_CONTEXT Context = Argument;
+    PLIST_ENTRY             ListEntry;
+
+    LogPrintf(LOG_LEVEL_INFO,
+              "SUSPEND: EARLY (%u)\n", Cpu);
+
+    if (!Context->Success || Cpu != 0)
+        return;
+
+    //
+    // No lock is required here as the VM is single-threaded with interrupts
+    // disabled.
+    //
+
+    Context->Count++;
+
+    HypercallPopulate();
+
+    UnplugDevices();
+
+    for (ListEntry = Context->EarlyList.Flink;
+         ListEntry != &Context->EarlyList;
+         ListEntry = ListEntry->Flink) {
+        PXENBUS_SUSPEND_CALLBACK  Callback;
+
+        Callback = CONTAINING_RECORD(ListEntry,
+                                     XENBUS_SUSPEND_CALLBACK,
+                                     ListEntry);
+        Callback->Function(Callback->Argument);
+    }
+}
+
+static VOID
+SuspendLate(
+    IN  PVOID               Argument,
+    IN  ULONG               Cpu
+    )
+{
+    PXENBUS_SUSPEND_CONTEXT Context = Argument;
+    PLIST_ENTRY             ListEntry;
+
+    LogPrintf(LOG_LEVEL_INFO,
+              "SUSPEND: LATE (%u)\n", Cpu);
+
+    if (!Context->Success || Cpu != 0)
+        return;
+
+    // No lock is required here as the VM is single-threaded
+
+    for (ListEntry = Context->LateList.Flink;
+         ListEntry != &Context->LateList;
+         ListEntry = ListEntry->Flink) {
+        PXENBUS_SUSPEND_CALLBACK  Callback;
+
+        Callback = CONTAINING_RECORD(ListEntry,
+                                     XENBUS_SUSPEND_CALLBACK,
+                                     ListEntry);
+        Callback->Function(Callback->Argument);
+    }
+}
+
 NTSTATUS
 #pragma prefast(suppress:28167) // Function changes IRQL
 SuspendTrigger(
@@ -195,7 +263,7 @@ SuspendTrigger(
     LogPrintf(LOG_LEVEL_INFO,
               "SUSPEND: ====>\n");
 
-    SyncCapture();
+    SyncCapture(Context, SuspendEarly, SuspendLate);
     SyncDisableInterrupts();
 
     __SuspendLogTimers("PRE-SUSPEND");
@@ -209,43 +277,9 @@ SuspendTrigger(
 
     __SuspendLogTimers("POST-SUSPEND");
 
-    if (NT_SUCCESS(status)) {
-        PLIST_ENTRY ListEntry;
-
-        Context->Count++;
-
-        HypercallPopulate();
-
-        UnplugDevices();
-
-        for (ListEntry = Context->EarlyList.Flink;
-             ListEntry != &Context->EarlyList;
-             ListEntry = ListEntry->Flink) {
-            PXENBUS_SUSPEND_CALLBACK  Callback;
-
-            Callback = CONTAINING_RECORD(ListEntry, XENBUS_SUSPEND_CALLBACK, ListEntry);
-            Callback->Function(Callback->Argument);
-        }
-    }
+    Context->Success = NT_SUCCESS(status) ? TRUE : FALSE;
 
     SyncEnableInterrupts();
-
-    // No lock is required here as the VM is single-threaded until
-    // SyncRelease() is called.
-
-    if (NT_SUCCESS(status)) {
-        PLIST_ENTRY ListEntry;
-
-        for (ListEntry = Context->LateList.Flink;
-             ListEntry != &Context->LateList;
-             ListEntry = ListEntry->Flink) {
-            PXENBUS_SUSPEND_CALLBACK  Callback;
-
-            Callback = CONTAINING_RECORD(ListEntry, XENBUS_SUSPEND_CALLBACK, ListEntry);
-            Callback->Function(Callback->Argument);
-        }
-    }
-
     SyncRelease();
 
     LogPrintf(LOG_LEVEL_INFO, "SUSPEND: <====\n");

diff --git a/src/xenbus/sync.c b/src/xenbus/sync.c
index 661aabe54a0d1104201365119fb640732a438ef6..615eca84c460f6dd67634af881bc23235241e4af 100644 (file)
--- a/src/xenbus/sync.c
+++ b/src/xenbus/sync.c
@@ -86,6 +86,9 @@ typedef struct  _SYNC_PROCESSOR {
 } SYNC_PROCESSOR, *PSYNC_PROCESSOR;
 
 typedef struct  _SYNC_CONTEXT {
+    PVOID               Argument;
+    SYNC_CALLBACK       Early;
+    SYNC_CALLBACK       Late;
     ULONG               Sequence;
     LONG                ProcessorCount;
     LONG                CompletionCount;
@@ -108,10 +111,13 @@ __SyncAcquire(
 
 static FORCEINLINE VOID
 __SyncRelease(
-    IN  LONG    Index
+    VOID
     )
 {
-    LONG        Old;
+    LONG    Old;
+    LONG    Index;
+
+    Index = KeGetCurrentProcessorNumberEx(NULL);
 
     Old = InterlockedExchange(&SyncOwner, -1);
     ASSERT3U(Old, ==, Index);
@@ -145,6 +151,9 @@ SyncWorker(
 
     InterruptsDisabled = FALSE;
     Index = KeGetCurrentProcessorNumberEx(&ProcNumber);
+
+    ASSERT(SyncOwner >= 0 && Index != (ULONG)SyncOwner);
+
     Processor = &Context->Processor[Index];
 
     Trace("====> (%u:%u)\n", ProcNumber.Group, ProcNumber.Number);
@@ -153,8 +162,12 @@ SyncWorker(
     for (;;) {
         ULONG   Sequence;
 
-        if (Processor->Exit)
+        if (Processor->Exit) {
+            if (Context->Late != NULL)
+                Context->Late(Context->Argument, Index);
+
             break;
+        }
 
         if (Processor->DisableInterrupts == InterruptsDisabled) {
             _mm_pause();
@@ -210,6 +223,9 @@ SyncWorker(
         } else {
             InterruptsDisabled = FALSE;
 
+            if (Context->Early != NULL)
+                Context->Early(Context->Argument, Index);
+
             _enable();
 
 #pragma prefast(suppress:28138) // Use constant rather than variable
@@ -236,7 +252,9 @@ __drv_maxIRQL(DISPATCH_LEVEL)
 __drv_raisesIRQL(DISPATCH_LEVEL)
 VOID
 SyncCapture(
-    VOID
+    IN  PVOID           Argument OPTIONAL,
+    IN  SYNC_CALLBACK   Early OPTIONAL,
+    IN  SYNC_CALLBACK   Late OPTIONAL
     )
 {
     PSYNC_CONTEXT       Context = SyncContext;
@@ -257,6 +275,10 @@ SyncCapture(
 
     ASSERT(IsZeroMemory(Context, PAGE_SIZE));
 
+    Context->Argument = Argument;
+    Context->Early = Early;
+    Context->Late = Late;
+
     Context->Sequence++;
     Context->CompletionCount = 0;
 
@@ -304,6 +326,8 @@ SyncDisableInterrupts(
 
     Trace("====>\n");
 
+    ASSERT(SyncOwner >= 0);
+
     Context->Sequence++;
     Context->CompletionCount = 0;
 
@@ -368,6 +392,11 @@ SyncEnableInterrupts(
     KIRQL           Irql;
     LONG            Index;
 
+    ASSERT(SyncOwner >= 0);
+
+    if (Context->Early != NULL)
+        Context->Early(Context->Argument, SyncOwner);
+
     _enable();
 
     Irql = KeGetCurrentIrql();
@@ -409,6 +438,11 @@ SyncRelease(
 
     Trace("====>\n");
 
+    ASSERT(SyncOwner >= 0);
+
+    if (Context->Late != NULL)
+        Context->Late(Context->Argument, SyncOwner);
+
     Context->Sequence++;
     Context->CompletionCount = 0;
 
@@ -429,8 +463,7 @@ SyncRelease(
 
     RtlZeroMemory(Context, PAGE_SIZE);
 
-    Index = KeGetCurrentProcessorNumberEx(NULL);
-    __SyncRelease(Index);
+    __SyncRelease();
 
     Trace("<====\n");
 }

diff --git a/src/xenbus/sync.h b/src/xenbus/sync.h
index affbf0f9f0d538b38ed683bbcb64834dd44fc649..c4b172107513b6ae3e09789e7698965c8b2ae3c3 100644 (file)
--- a/src/xenbus/sync.h
+++ b/src/xenbus/sync.h
@@ -34,12 +34,20 @@
 
 #include <ntddk.h>
 
+typedef VOID
+(*SYNC_CALLBACK)(
+    IN  PVOID   Arguement,
+    IN  ULONG   Cpu
+    );
+
 extern
 __drv_maxIRQL(DISPATCH_LEVEL)
 __drv_raisesIRQL(DISPATCH_LEVEL)
 VOID
 SyncCapture(
-    VOID
+    IN  PVOID           Argument OPTIONAL,
+    IN  SYNC_CALLBACK   Early OPTIONAL,
+    IN  SYNC_CALLBACK   Late OPTIONAL
     );
 
 extern