win-pvdrivers

changeset 622:f905eb3f0545

Shut down threads properly on hibernate.
Added some debug to try and find where the hang happens when the verifier is enabled
Added the suspend event channel to allow triggering a suspend much earlier
author James Harper <james.harper@bendigoit.com.au>
date Wed Aug 05 19:09:55 2009 +1000 (2009-08-05)
parents 9c7b7fbacd7f
children b79b67b46d0c
files xenpci/xenbus.c xenpci/xenpci.c xenpci/xenpci.h xenpci/xenpci_fdo.c
line diff
     1.1 --- a/xenpci/xenbus.c	Wed Jul 29 12:43:36 2009 -0400
     1.2 +++ b/xenpci/xenbus.c	Wed Aug 05 19:09:55 2009 +1000
     1.3 @@ -286,7 +286,7 @@ XenBus_Init(PXENPCI_DEVICE_DATA xpdd)
     1.4      
     1.5    KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n"));
     1.6  
     1.7 -  ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
     1.8 +  ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
     1.9  
    1.10    ExInitializeFastMutex(&xpdd->xb_request_mutex);
    1.11    ExInitializeFastMutex(&xpdd->xb_watch_mutex);
    1.12 @@ -307,29 +307,38 @@ XenBus_Init(PXENPCI_DEVICE_DATA xpdd)
    1.13    }
    1.14    EvtChn_BindDpc(xpdd, xpdd->xen_store_evtchn, XenBus_Dpc, xpdd);
    1.15  
    1.16 +KdPrint((__DRIVER_NAME "     A\n"));
    1.17    status = PsCreateSystemThread(&thread_handle, THREAD_ALL_ACCESS, NULL, NULL, NULL, XenBus_ReadThreadProc, xpdd);
    1.18    if (!NT_SUCCESS(status))
    1.19    {
    1.20      KdPrint((__DRIVER_NAME "     Could not start read thread\n"));
    1.21      return status;
    1.22    }
    1.23 +KdPrint((__DRIVER_NAME "     B\n"));  
    1.24 +  status = ObReferenceObjectByHandle(thread_handle, THREAD_ALL_ACCESS, NULL, KernelMode, &xpdd->XenBus_ReadThread, NULL);
    1.25 +KdPrint((__DRIVER_NAME "     C\n"));
    1.26 +  ZwClose(thread_handle);
    1.27    
    1.28 -  status = ObReferenceObjectByHandle(thread_handle, THREAD_ALL_ACCESS, NULL, KernelMode, &xpdd->XenBus_ReadThread, NULL);
    1.29 -  ZwClose(thread_handle);
    1.30 +KdPrint((__DRIVER_NAME "     D\n"));
    1.31    if (!NT_SUCCESS(status))
    1.32    {
    1.33      KdPrint((__DRIVER_NAME "     ObReferenceObjectByHandle(XenBus_ReadThread) = %08x\n", status));
    1.34      return status;
    1.35    }
    1.36 +KdPrint((__DRIVER_NAME "     E\n"));
    1.37  
    1.38    status = PsCreateSystemThread(&thread_handle, THREAD_ALL_ACCESS, NULL, NULL, NULL, XenBus_WatchThreadProc, xpdd);
    1.39 +KdPrint((__DRIVER_NAME "     F\n"));
    1.40    if (!NT_SUCCESS(status))
    1.41    {
    1.42      KdPrint((__DRIVER_NAME " Could not start watch thread\n"));
    1.43      return status;
    1.44    }
    1.45 +KdPrint((__DRIVER_NAME "     G\n"));
    1.46    status = ObReferenceObjectByHandle(thread_handle, THREAD_ALL_ACCESS, NULL, KernelMode, &xpdd->XenBus_WatchThread, NULL);
    1.47 +KdPrint((__DRIVER_NAME "     H\n"));
    1.48    ZwClose(thread_handle);
    1.49 +KdPrint((__DRIVER_NAME "     I\n"));
    1.50    if (!NT_SUCCESS(status))
    1.51    {
    1.52      KdPrint((__DRIVER_NAME "     ObReferenceObjectByHandle(XenBus_WatchThread) = %08x\n", status));
    1.53 @@ -371,10 +380,9 @@ XenBus_SendRemWatch(
    1.54  }
    1.55  
    1.56  NTSTATUS
    1.57 -XenBus_StopThreads(PXENPCI_DEVICE_DATA xpdd)
    1.58 +XenBus_Halt(PXENPCI_DEVICE_DATA xpdd)
    1.59  {
    1.60    NTSTATUS status;
    1.61 -  //KWAIT_BLOCK WaitBlockArray[2];
    1.62    int i;
    1.63    LARGE_INTEGER timeout;
    1.64  
    1.65 @@ -391,21 +399,24 @@ XenBus_StopThreads(PXENPCI_DEVICE_DATA x
    1.66    xpdd->XenBus_ShuttingDown = TRUE;
    1.67    KeMemoryBarrier();
    1.68  
    1.69 +  KeSetEvent(&xpdd->XenBus_WatchThreadEvent, IO_NO_INCREMENT, FALSE);
    1.70    KeSetEvent(&xpdd->XenBus_ReadThreadEvent, IO_NO_INCREMENT, FALSE);
    1.71 -  KeSetEvent(&xpdd->XenBus_WatchThreadEvent, IO_NO_INCREMENT, FALSE);
    1.72    
    1.73    timeout.QuadPart = (LONGLONG)-1 * 1000 * 1000 * 10;
    1.74 +  while ((status = KeWaitForSingleObject(xpdd->XenBus_WatchThread, Executive, KernelMode, FALSE, &timeout)) != STATUS_SUCCESS)
    1.75 +  {
    1.76 +    timeout.QuadPart = (LONGLONG)-1 * 1000 * 1000 * 10;
    1.77 +    KdPrint((__DRIVER_NAME "     Waiting for XenBus_WatchThread to stop\n"));
    1.78 +  }
    1.79 +  ObDereferenceObject(xpdd->XenBus_WatchThread);
    1.80 +
    1.81 +  timeout.QuadPart = (LONGLONG)-1 * 1000 * 1000 * 10;
    1.82    while ((status = KeWaitForSingleObject(xpdd->XenBus_ReadThread, Executive, KernelMode, FALSE, &timeout)) != STATUS_SUCCESS)
    1.83    {
    1.84      timeout.QuadPart = (LONGLONG)-1 * 1000 * 1000 * 10;
    1.85 +    KdPrint((__DRIVER_NAME "     Waiting for XenBus_ReadThread to stop\n"));
    1.86    }
    1.87    ObDereferenceObject(xpdd->XenBus_ReadThread);
    1.88 -  timeout.QuadPart = (LONGLONG)-1 * 1000 * 1000 * 10;
    1.89 -  while ((status = KeWaitForSingleObject(xpdd->XenBus_WatchThread, Executive, KernelMode, FALSE, &timeout)) != STATUS_SUCCESS)
    1.90 -  {
    1.91 -    timeout.QuadPart = (LONGLONG)-1 * 1000 * 1000 * 10;
    1.92 -  }
    1.93 -  ObDereferenceObject(xpdd->XenBus_WatchThread);
    1.94    
    1.95    xpdd->XenBus_ShuttingDown = FALSE;
    1.96  
    1.97 @@ -470,6 +481,7 @@ XenBus_ReadThreadProc(PVOID StartContext
    1.98    char *path, *token;
    1.99    PXENPCI_DEVICE_DATA xpdd = StartContext;
   1.100  
   1.101 +  FUNCTION_ENTER();
   1.102    for(;;)
   1.103    {
   1.104      KeWaitForSingleObject(&xpdd->XenBus_ReadThreadEvent, Executive, KernelMode, FALSE, NULL);
   1.105 @@ -535,6 +547,7 @@ XenBus_ReadThreadProc(PVOID StartContext
   1.106        }
   1.107      }
   1.108    }
   1.109 +  FUNCTION_EXIT();
   1.110  }
   1.111  
   1.112  static DDKAPI void
   1.113 @@ -620,7 +633,8 @@ XenBus_Suspend(PXENPCI_DEVICE_DATA xpdd)
   1.114    }
   1.115  
   1.116    // need to synchronise with readthread here too to ensure that it won't do anything silly
   1.117 -  
   1.118 +  MmUnmapIoSpace(xpdd->xen_store_interface, PAGE_SIZE);
   1.119 +
   1.120    return STATUS_SUCCESS;
   1.121  }
   1.122  
     2.1 --- a/xenpci/xenpci.c	Wed Jul 29 12:43:36 2009 -0400
     2.2 +++ b/xenpci/xenpci.c	Wed Aug 05 19:09:55 2009 +1000
     2.3 @@ -116,6 +116,7 @@ XenPci_EvtDeviceAdd_XenPci(WDFDRIVER dri
     2.4    }
     2.5  
     2.6    xpdd = GetXpdd(device);
     2.7 +  xpdd->wdf_device = device;
     2.8    xpdd->child_list = WdfFdoGetDefaultChildList(device);
     2.9  
    2.10    WdfCollectionCreate(WDF_NO_OBJECT_ATTRIBUTES, &xpdd->veto_devices);
     3.1 --- a/xenpci/xenpci.h	Wed Jul 29 12:43:36 2009 -0400
     3.2 +++ b/xenpci/xenpci.h	Wed Aug 05 19:09:55 2009 +1000
     3.3 @@ -115,7 +115,7 @@ typedef struct _XENBUS_WATCH_ENTRY {
     3.4  #define SUSPEND_STATE_RESUMING  3 /* we are the other side of the suspend and things are starting to get back to normal */
     3.5  
     3.6  typedef struct {  
     3.7 -  //XENPCI_COMMON common;
     3.8 +  WDFDEVICE wdf_device;
     3.9    
    3.10    BOOLEAN XenBus_ShuttingDown;
    3.11    
    3.12 @@ -176,6 +176,7 @@ typedef struct {
    3.13  #define BALLOON_UNITS (1024 * 1024) /* 1MB */
    3.14    PKTHREAD balloon_thread;
    3.15    KEVENT balloon_event;
    3.16 +  BOOLEAN balloon_shutdown;
    3.17    ULONG initial_memory;
    3.18    ULONG current_memory;
    3.19    ULONG target_memory;
    3.20 @@ -188,7 +189,9 @@ typedef struct {
    3.21    struct xsd_sockmsg *xb_reply;
    3.22    
    3.23    WDFCHILDLIST child_list;
    3.24 -  
    3.25 +
    3.26 +  KSPIN_LOCK suspend_lock;  
    3.27 +  evtchn_port_t suspend_evtchn;
    3.28    int suspend_state;
    3.29    
    3.30    UNICODE_STRING legacy_interface_name;
    3.31 @@ -469,17 +472,11 @@ XenBus_RemWatch(PVOID Context, xenbus_tr
    3.32  NTSTATUS
    3.33  XenBus_Init(PXENPCI_DEVICE_DATA xpdd);
    3.34  NTSTATUS
    3.35 -XenBus_Close(PXENPCI_DEVICE_DATA xpdd);
    3.36 -NTSTATUS
    3.37 -XenBus_Start(PXENPCI_DEVICE_DATA xpdd);
    3.38 -NTSTATUS
    3.39 -XenBus_Stop(PXENPCI_DEVICE_DATA xpdd);
    3.40 +XenBus_Halt(PXENPCI_DEVICE_DATA xpdd);
    3.41  NTSTATUS
    3.42  XenBus_Suspend(PXENPCI_DEVICE_DATA xpdd);
    3.43  NTSTATUS
    3.44  XenBus_Resume(PXENPCI_DEVICE_DATA xpdd);
    3.45 -NTSTATUS
    3.46 -XenBus_StopThreads(PXENPCI_DEVICE_DATA xpdd);
    3.47  
    3.48  PHYSICAL_ADDRESS
    3.49  XenPci_AllocMMIO(PXENPCI_DEVICE_DATA xpdd, ULONG len);
     4.1 --- a/xenpci/xenpci_fdo.c	Wed Jul 29 12:43:36 2009 -0400
     4.2 +++ b/xenpci/xenpci_fdo.c	Wed Aug 05 19:09:55 2009 +1000
     4.3 @@ -134,7 +134,7 @@ XenPci_Init(PXENPCI_DEVICE_DATA xpdd)
     4.4    KdPrint((__DRIVER_NAME "     gpfn = %x\n", xatp.gpfn));
     4.5    ret = HYPERVISOR_memory_op(xpdd, XENMEM_add_to_physmap, &xatp);
     4.6    KdPrint((__DRIVER_NAME "     hypervisor memory op (XENMAPSPACE_shared_info) ret = %d\n", ret));
     4.7 -  
     4.8 +
     4.9    FUNCTION_EXIT();
    4.10  
    4.11    return STATUS_SUCCESS;
    4.12 @@ -249,6 +249,8 @@ XenPci_BalloonThreadProc(PVOID StartCont
    4.13        ptimeout = NULL;
    4.14      }
    4.15      KeWaitForSingleObject(&xpdd->balloon_event, Executive, KernelMode, FALSE, ptimeout);
    4.16 +    if (xpdd->balloon_shutdown)
    4.17 +      PsTerminateSystemThread(0);
    4.18  //TODO: initiate shutdown here
    4.19      KdPrint((__DRIVER_NAME "     Got balloon event, current = %d, target = %d\n", xpdd->current_memory, xpdd->target_memory));
    4.20      /* not really worried about races here, but cache target so we only read it once */
    4.21 @@ -389,6 +391,26 @@ XenPci_SuspendN(PVOID context)
    4.22    FUNCTION_EXIT();
    4.23  }
    4.24  
    4.25 +static VOID
    4.26 +XenPci_SuspendEvtDpc(PVOID context);
    4.27 +static NTSTATUS
    4.28 +XenPci_ConnectSuspendEvt(PXENPCI_DEVICE_DATA xpdd);
    4.29 +
    4.30 +/* called at PASSIVE_LEVEL */
    4.31 +static NTSTATUS
    4.32 +XenPci_ConnectSuspendEvt(PXENPCI_DEVICE_DATA xpdd)
    4.33 +{
    4.34 +  CHAR path[128];
    4.35 +
    4.36 +  xpdd->suspend_evtchn = EvtChn_AllocUnbound(xpdd, 0);
    4.37 +  KdPrint((__DRIVER_NAME "     suspend event channel = %d\n", xpdd->suspend_evtchn));
    4.38 +  RtlStringCbPrintfA(path, ARRAY_SIZE(path), "device/suspend/event-channel");
    4.39 +  XenBus_Printf(xpdd, XBT_NIL, path, "%d", xpdd->suspend_evtchn);
    4.40 +  EvtChn_BindDpc(xpdd, xpdd->suspend_evtchn, XenPci_SuspendEvtDpc, xpdd->wdf_device);
    4.41 +  
    4.42 +  return STATUS_SUCCESS;
    4.43 +}
    4.44 +
    4.45  /* Called at PASSIVE_LEVEL */
    4.46  static VOID DDKAPI
    4.47  XenPci_SuspendResume(WDFWORKITEM workitem)
    4.48 @@ -425,6 +447,8 @@ XenPci_SuspendResume(WDFWORKITEM workite
    4.49      xpdd->suspend_state = SUSPEND_STATE_RESUMING;
    4.50      XenBus_Resume(xpdd);
    4.51  
    4.52 +    XenPci_ConnectSuspendEvt(xpdd);
    4.53 +
    4.54      WdfChildListBeginIteration(child_list, &child_iterator);
    4.55      while ((status = WdfChildListRetrieveNextDevice(child_list, &child_iterator, &child_device, NULL)) == STATUS_SUCCESS)
    4.56      {
    4.57 @@ -439,6 +463,26 @@ XenPci_SuspendResume(WDFWORKITEM workite
    4.58    FUNCTION_EXIT();
    4.59  }
    4.60  
    4.61 +/* called at DISPATCH_LEVEL */
    4.62 +static VOID
    4.63 +XenPci_SuspendEvtDpc(PVOID context)
    4.64 +{
    4.65 +  NTSTATUS status;
    4.66 +  WDFDEVICE device = context;
    4.67 +  //KIRQL old_irql;
    4.68 +  WDF_OBJECT_ATTRIBUTES attributes;
    4.69 +  WDF_WORKITEM_CONFIG workitem_config;
    4.70 +  WDFWORKITEM workitem;
    4.71 +
    4.72 +  KdPrint((__DRIVER_NAME "     Suspend detected via Dpc\n"));
    4.73 +  WDF_WORKITEM_CONFIG_INIT(&workitem_config, XenPci_SuspendResume);
    4.74 +  WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
    4.75 +  attributes.ParentObject = device;
    4.76 +  status = WdfWorkItemCreate(&workitem_config, &attributes, &workitem);
    4.77 +  // TODO: check status here
    4.78 +  WdfWorkItemEnqueue(workitem);
    4.79 +}
    4.80 +
    4.81  static void
    4.82  XenPci_ShutdownHandler(char *path, PVOID context)
    4.83  {
    4.84 @@ -665,6 +709,7 @@ XenPci_EvtDeviceD0EntryPostInterruptsEna
    4.85    domid_t domid = DOMID_SELF;
    4.86    ULONG ret;
    4.87    xen_ulong_t *max_ram_page;
    4.88 +  HANDLE thread_handle;
    4.89  
    4.90    UNREFERENCED_PARAMETER(previous_state);
    4.91  
    4.92 @@ -672,6 +717,8 @@ XenPci_EvtDeviceD0EntryPostInterruptsEna
    4.93    
    4.94    XenBus_Init(xpdd);
    4.95  
    4.96 +  XenPci_ConnectSuspendEvt(xpdd);
    4.97 +  
    4.98    response = XenBus_AddWatch(xpdd, XBT_NIL, SYSRQ_PATH, XenPci_SysrqHandler, xpdd);
    4.99    
   4.100    response = XenBus_AddWatch(xpdd, XBT_NIL, SHUTDOWN_PATH, XenPci_ShutdownHandler, device);
   4.101 @@ -696,12 +743,15 @@ XenPci_EvtDeviceD0EntryPostInterruptsEna
   4.102      }
   4.103      KdPrint((__DRIVER_NAME "     Initial Memory Value = %d (%s)\n", xpdd->initial_memory, value));
   4.104      KeInitializeEvent(&xpdd->balloon_event, SynchronizationEvent, FALSE);
   4.105 -    status = PsCreateSystemThread(&xpdd->balloon_thread, THREAD_ALL_ACCESS, NULL, NULL, NULL, XenPci_BalloonThreadProc, xpdd);
   4.106 +    xpdd->balloon_shutdown = FALSE;
   4.107 +    status = PsCreateSystemThread(&thread_handle, THREAD_ALL_ACCESS, NULL, NULL, NULL, XenPci_BalloonThreadProc, xpdd);
   4.108      if (!NT_SUCCESS(status))
   4.109      {
   4.110        KdPrint((__DRIVER_NAME "     Could not start balloon thread\n"));
   4.111 -      //return status;
   4.112 -    }    
   4.113 +      return status;
   4.114 +    }
   4.115 +    status = ObReferenceObjectByHandle(thread_handle, THREAD_ALL_ACCESS, NULL, KernelMode, &xpdd->balloon_thread, NULL);
   4.116 +    ZwClose(thread_handle);
   4.117    }
   4.118    response = XenBus_AddWatch(xpdd, XBT_NIL, BALLOON_PATH, XenPci_BalloonHandler, device);
   4.119  
   4.120 @@ -714,8 +764,8 @@ NTSTATUS
   4.121  XenPci_EvtDeviceD0ExitPreInterruptsDisabled(WDFDEVICE device, WDF_POWER_DEVICE_STATE target_state)
   4.122  {
   4.123    NTSTATUS status = STATUS_SUCCESS;
   4.124 -  
   4.125 -  UNREFERENCED_PARAMETER(device);
   4.126 +  PXENPCI_DEVICE_DATA xpdd = GetXpdd(device);
   4.127 +  LARGE_INTEGER timeout;
   4.128    
   4.129    FUNCTION_ENTER();
   4.130    
   4.131 @@ -744,6 +794,19 @@ XenPci_EvtDeviceD0ExitPreInterruptsDisab
   4.132      break;  
   4.133    }
   4.134    
   4.135 +  xpdd->balloon_shutdown = TRUE;
   4.136 +  KeSetEvent(&xpdd->balloon_event, IO_NO_INCREMENT, FALSE);
   4.137 +  
   4.138 +  timeout.QuadPart = (LONGLONG)-1 * 1000 * 1000 * 10;
   4.139 +  while ((status = KeWaitForSingleObject(xpdd->balloon_thread, Executive, KernelMode, FALSE, &timeout)) != STATUS_SUCCESS)
   4.140 +  {
   4.141 +    timeout.QuadPart = (LONGLONG)-1 * 1000 * 1000 * 10;
   4.142 +    KdPrint((__DRIVER_NAME "     Waiting for balloon thread to stop\n"));
   4.143 +  }
   4.144 +  ObDereferenceObject(xpdd->balloon_thread);
   4.145 +
   4.146 +  XenBus_Halt(xpdd);
   4.147 +  
   4.148    FUNCTION_EXIT();
   4.149    
   4.150    return status;