#include "adapter.h"
#include "driver.h"
#include "registry.h"
-#include "pdo.h"
+#include "target.h"
#include "srbext.h"
#include "thread.h"
#include "buffer.h"
// Targets
KSPIN_LOCK TargetLock;
- PXENVBD_PDO Targets[XENVBD_MAX_TARGETS];
+ PXENVBD_TARGET Targets[XENVBD_MAX_TARGETS];
// Target Enumeration
PXENVBD_THREAD ScanThread;
}
__checkReturn
-static FORCEINLINE PXENVBD_PDO
-__AdapterGetPdoAlways(
+static FORCEINLINE PXENVBD_TARGET
+__AdapterGetTargetAlways(
__in PXENVBD_ADAPTER Adapter,
__in ULONG TargetId,
__in PCHAR Caller
)
{
- PXENVBD_PDO Pdo;
+ PXENVBD_TARGET Target;
KIRQL Irql;
ASSERT3U(TargetId, <, XENVBD_MAX_TARGETS);
KeAcquireSpinLock(&Adapter->TargetLock, &Irql);
- Pdo = Adapter->Targets[TargetId];
- if (Pdo) {
- __PdoReference(Pdo, Caller);
+ Target = Adapter->Targets[TargetId];
+ if (Target) {
+ __TargetReference(Target, Caller);
}
KeReleaseSpinLock(&Adapter->TargetLock, Irql);
- return Pdo;
+ return Target;
}
__checkReturn
-static FORCEINLINE PXENVBD_PDO
-___AdapterGetPdo(
+static FORCEINLINE PXENVBD_TARGET
+___AdapterGetTarget(
__in PXENVBD_ADAPTER Adapter,
__in ULONG TargetId,
__in PCHAR Caller
)
{
- PXENVBD_PDO Pdo = NULL;
+ PXENVBD_TARGET Target = NULL;
KIRQL Irql;
ASSERT3U(TargetId, <, XENVBD_MAX_TARGETS);
KeAcquireSpinLock(&Adapter->TargetLock, &Irql);
if (Adapter->Targets[TargetId] &&
- __PdoReference(Adapter->Targets[TargetId], Caller) > 0) {
- Pdo = Adapter->Targets[TargetId];
+ __TargetReference(Adapter->Targets[TargetId], Caller) > 0) {
+ Target = Adapter->Targets[TargetId];
}
KeReleaseSpinLock(&Adapter->TargetLock, Irql);
- return Pdo;
+ return Target;
}
-#define __AdapterGetPdo(f, t) ___AdapterGetPdo(f, t, __FUNCTION__)
+#define __AdapterGetTarget(f, t) ___AdapterGetTarget(f, t, __FUNCTION__)
BOOLEAN
-AdapterLinkPdo(
+AdapterLinkTarget(
__in PXENVBD_ADAPTER Adapter,
- __in PXENVBD_PDO Pdo
+ __in PXENVBD_TARGET Target
)
{
KIRQL Irql;
- PXENVBD_PDO Current;
+ PXENVBD_TARGET Current;
BOOLEAN Result = FALSE;
- ULONG TargetId = PdoGetTargetId(Pdo);
+ ULONG TargetId = TargetGetTargetId(Target);
KeAcquireSpinLock(&Adapter->TargetLock, &Irql);
Current = Adapter->Targets[TargetId];
if (Adapter->Targets[TargetId] == NULL) {
- Adapter->Targets[TargetId] = Pdo;
+ Adapter->Targets[TargetId] = Target;
Result = TRUE;
}
KeReleaseSpinLock(&Adapter->TargetLock, Irql);
if (!Result) {
- Warning("Target[%d] : Current 0x%p, New 0x%p\n", TargetId, Current, Pdo);
+ Warning("Target[%d] : Current 0x%p, New 0x%p\n", TargetId, Current, Target);
}
return Result;
}
BOOLEAN
-AdapterUnlinkPdo(
+AdapterUnlinkTarget(
__in PXENVBD_ADAPTER Adapter,
- __in PXENVBD_PDO Pdo
+ __in PXENVBD_TARGET Target
)
{
KIRQL Irql;
- PXENVBD_PDO Current;
+ PXENVBD_TARGET Current;
BOOLEAN Result = FALSE;
- ULONG TargetId = PdoGetTargetId(Pdo);
+ ULONG TargetId = TargetGetTargetId(Target);
KeAcquireSpinLock(&Adapter->TargetLock, &Irql);
Current = Adapter->Targets[TargetId];
- if (Adapter->Targets[TargetId] == Pdo) {
+ if (Adapter->Targets[TargetId] == Target) {
Adapter->Targets[TargetId] = NULL;
Result = TRUE;
}
KeReleaseSpinLock(&Adapter->TargetLock, Irql);
if (!Result) {
- Warning("Target[%d] : Current 0x%p, Expected 0x%p\n", TargetId, Current, Pdo);
+ Warning("Target[%d] : Current 0x%p, Expected 0x%p\n", TargetId, Current, Target);
}
return Result;
}
BufferDebugCallback(&Adapter->Debug);
for (TargetId = 0; TargetId < XENVBD_MAX_TARGETS; ++TargetId) {
- // no need to use __AdapterGetPdo (which is locked at DISPATCH) as called at HIGH_LEVEL
- PXENVBD_PDO Pdo = Adapter->Targets[TargetId];
- if (Pdo == NULL)
+ // no need to use __AdapterGetTarget (which is locked at DISPATCH) as called at HIGH_LEVEL
+ PXENVBD_TARGET Target = Adapter->Targets[TargetId];
+ if (Target == NULL)
continue;
XENBUS_DEBUG(Printf, &Adapter->Debug,
"ADAPTER: ====> Target[%-3d] : 0x%p\n",
- TargetId, Pdo);
+ TargetId, Target);
// call Target's debug callback directly
- PdoDebugCallback(Pdo, &Adapter->Debug);
+ TargetDebugCallback(Target, &Adapter->Debug);
XENBUS_DEBUG(Printf, &Adapter->Debug,
"ADAPTER: <==== Target[%-3d] : 0x%p\n",
- TargetId, Pdo);
+ TargetId, Target);
}
Adapter->MaximumSrbs = Adapter->CurrentSrbs;
}
__checkReturn
static FORCEINLINE BOOLEAN
-__AdapterIsPdoUnplugged(
+__AdapterIsTargetUnplugged(
__in PXENVBD_ADAPTER Adapter,
__in PCHAR Enumerator,
__in PCHAR Device,
ULONG TargetId;
PANSI_STRING Device;
ULONG Index;
- PXENVBD_PDO Pdo;
+ PXENVBD_TARGET Target;
*NeedInvalidate = FALSE;
*NeedReboot = FALSE;
for (TargetId = 0; TargetId < XENVBD_MAX_TARGETS; ++TargetId) {
BOOLEAN Missing = TRUE;
- Pdo = __AdapterGetPdo(Adapter, TargetId);
- if (Pdo == NULL)
+ Target = __AdapterGetTarget(Adapter, TargetId);
+ if (Target == NULL)
continue;
for (Index = 0; Devices[Index].Buffer != NULL; ++Index) {
}
}
- if (Missing && !PdoIsMissing(Pdo)) {
- PdoSetMissing(Pdo, "Device Disappeared");
- if (PdoGetDevicePnpState(Pdo) == Present)
- PdoSetDevicePnpState(Pdo, Deleted);
+ if (Missing && !TargetIsMissing(Target)) {
+ TargetSetMissing(Target, "Device Disappeared");
+ if (TargetGetDevicePnpState(Target) == Present)
+ TargetSetDevicePnpState(Target, Deleted);
else
*NeedInvalidate = TRUE;
}
- if (PdoGetDevicePnpState(Pdo) == Deleted) {
- PdoDereference(Pdo);
- PdoDestroy(Pdo);
+ if (TargetGetDevicePnpState(Target) == Deleted) {
+ TargetDereference(Target);
+ TargetDestroy(Target);
} else {
- PdoDereference(Pdo);
+ TargetDereference(Target);
}
}
continue;
}
- Pdo = __AdapterGetPdo(Adapter, TargetId);
- if (Pdo) {
- PdoDereference(Pdo);
+ Target = __AdapterGetTarget(Adapter, TargetId);
+ if (Target) {
+ TargetDereference(Target);
continue;
}
continue;
}
- if (!__AdapterIsPdoUnplugged(Adapter,
+ if (!__AdapterIsTargetUnplugged(Adapter,
AdapterEnum(Adapter),
Device->Buffer,
TargetId)) {
continue;
}
- if (PdoCreate(Adapter,
+ if (TargetCreate(Adapter,
Device->Buffer,
TargetId,
DeviceType)) {
if (!NT_SUCCESS(Status))
goto fail2;
- // Power UP any PDOs
+ // Power UP any TARGETs
for (TargetId = 0; TargetId < XENVBD_MAX_TARGETS; ++TargetId) {
- PXENVBD_PDO Pdo = __AdapterGetPdo(Adapter, TargetId);
- if (Pdo) {
- Status = PdoD3ToD0(Pdo);
- PdoDereference(Pdo);
+ PXENVBD_TARGET Target = __AdapterGetTarget(Adapter, TargetId);
+ if (Target) {
+ Status = TargetD3ToD0(Target);
+ TargetDereference(Target);
if (!NT_SUCCESS(Status))
goto fail3;
Error("Fail3\n");
for (TargetId = 0; TargetId < XENVBD_MAX_TARGETS; ++TargetId) {
- PXENVBD_PDO Pdo = __AdapterGetPdo(Adapter, TargetId);
- if (Pdo) {
- PdoD0ToD3(Pdo);
- PdoDereference(Pdo);
+ PXENVBD_TARGET Target = __AdapterGetTarget(Adapter, TargetId);
+ if (Target) {
+ TargetD0ToD3(Target);
+ TargetDereference(Target);
}
}
__AdapterD0ToD3(Adapter);
- // Power DOWN any PDOs
+ // Power DOWN any TARGETs
for (TargetId = 0; TargetId < XENVBD_MAX_TARGETS; ++TargetId) {
- PXENVBD_PDO Pdo = __AdapterGetPdo(Adapter, TargetId);
- if (Pdo) {
- PdoD0ToD3(Pdo);
- PdoDereference(Pdo);
+ PXENVBD_TARGET Target = __AdapterGetTarget(Adapter, TargetId);
+ if (Target) {
+ TargetD0ToD3(Target);
+ TargetDereference(Target);
}
}
// delete targets
for (TargetId = 0; TargetId < XENVBD_MAX_TARGETS; ++TargetId) {
- PXENVBD_PDO Pdo = __AdapterGetPdoAlways(Adapter, TargetId, __FUNCTION__);
- if (Pdo) {
- // Pdo may not be in Deleted state yet, force it as Adapter is terminating
- if (PdoGetDevicePnpState(Pdo) != Deleted)
- PdoSetDevicePnpState(Pdo, Deleted);
+ PXENVBD_TARGET Target = __AdapterGetTargetAlways(Adapter, TargetId, __FUNCTION__);
+ if (Target) {
+ // Target may not be in Deleted state yet, force it as Adapter is terminating
+ if (TargetGetDevicePnpState(Target) != Deleted)
+ TargetSetDevicePnpState(Target, Deleted);
// update missing (for debug output more than anything else
- PdoSetMissing(Pdo, "AdapterTerminate");
- // drop ref-count acquired in __AdapterGetPdo *before* destroying Pdo
- PdoDereference(Pdo);
- PdoDestroy(Pdo);
+ TargetSetMissing(Target, "AdapterTerminate");
+ // drop ref-count acquired in __AdapterGetTarget *before* destroying Target
+ TargetDereference(Target);
+ TargetDestroy(Target);
}
}
Verbose("====>\n");
for (TargetId = 0; TargetId < XENVBD_MAX_TARGETS; ++TargetId) {
- PXENVBD_PDO Pdo = __AdapterGetPdo(Adapter, TargetId);
- if (Pdo) {
- PdoReset(Pdo);
- PdoDereference(Pdo);
+ PXENVBD_TARGET Target = __AdapterGetTarget(Adapter, TargetId);
+ if (Target) {
+ TargetReset(Target);
+ TargetDereference(Target);
}
}
Verbose("<====\n");
)
{
if (!(Srb->SrbPnPFlags & SRB_PNP_FLAGS_ADAPTER_REQUEST)) {
- PXENVBD_PDO Pdo;
+ PXENVBD_TARGET Target;
- Pdo = __AdapterGetPdo(Adapter, Srb->TargetId);
- if (Pdo) {
- PdoSrbPnp(Pdo, Srb);
- PdoDereference(Pdo);
+ Target = __AdapterGetTarget(Adapter, Srb->TargetId);
+ if (Target) {
+ TargetSrbPnp(Target, Srb);
+ TargetDereference(Target);
}
}
}
__in PSCSI_REQUEST_BLOCK Srb
)
{
- PXENVBD_PDO Pdo;
+ PXENVBD_TARGET Target;
BOOLEAN CompleteSrb = TRUE;
- Pdo = __AdapterGetPdo(Adapter, Srb->TargetId);
- if (Pdo) {
- CompleteSrb = PdoStartIo(Pdo, Srb);
- PdoDereference(Pdo);
+ Target = __AdapterGetTarget(Adapter, Srb->TargetId);
+ if (Target) {
+ CompleteSrb = TargetStartIo(Target, Srb);
+ TargetDereference(Target);
}
if (CompleteSrb) {
return TRUE;
}
-static PXENVBD_PDO
-AdapterGetPdoFromDeviceObject(
+static PXENVBD_TARGET
+AdapterGetTargetFromDeviceObject(
__in PXENVBD_ADAPTER Adapter,
__in PDEVICE_OBJECT DeviceObject
)
ASSERT3P(DeviceObject, !=, NULL);
for (TargetId = 0; TargetId < XENVBD_MAX_TARGETS; ++TargetId) {
- PXENVBD_PDO Pdo = __AdapterGetPdo(Adapter, TargetId);
- if (Pdo) {
- if (PdoGetDeviceObject(Pdo) == DeviceObject)
- return Pdo;
- PdoDereference(Pdo);
+ PXENVBD_TARGET Target = __AdapterGetTarget(Adapter, TargetId);
+ if (Target) {
+ if (TargetGetDeviceObject(Target) == DeviceObject)
+ return Target;
+ TargetDereference(Target);
}
}
return NULL;
}
-static PXENVBD_PDO
-AdapterMapDeviceObjectToPdo(
+static PXENVBD_TARGET
+AdapterMapDeviceObjectToTarget(
__in PXENVBD_ADAPTER Adapter,
__in PDEVICE_OBJECT DeviceObject
)
{
- PXENVBD_PDO Pdo;
+ PXENVBD_TARGET Target;
KEVENT Complete;
PIRP Irp;
IO_STATUS_BLOCK StatusBlock;
if (!NT_SUCCESS(Status))
goto fail4;
- Pdo = __AdapterGetPdo(Adapter, TargetId);
- if (Pdo == NULL)
+ Target = __AdapterGetTarget(Adapter, TargetId);
+ if (Target == NULL)
goto fail5;
- PdoSetDeviceObject(Pdo, DeviceObject);
+ TargetSetDeviceObject(Target, DeviceObject);
ExFreePool(String);
- return Pdo;
+ return Target;
fail5:
fail4:
)
{
PIO_STACK_LOCATION Stack;
- PXENVBD_PDO Pdo;
+ PXENVBD_TARGET Target;
ASSERT3P(DeviceObject, !=, Adapter->DeviceObject);
- Pdo = AdapterGetPdoFromDeviceObject(Adapter, DeviceObject);
- if (Pdo != NULL) {
- return PdoDispatchPnp(Pdo, DeviceObject, Irp);
+ Target = AdapterGetTargetFromDeviceObject(Adapter, DeviceObject);
+ if (Target != NULL) {
+ return TargetDispatchPnp(Target, DeviceObject, Irp);
}
Stack = IoGetCurrentIrpStackLocation(Irp);
if (Stack->MinorFunction == IRP_MN_QUERY_ID &&
Stack->Parameters.QueryId.IdType == BusQueryDeviceID) {
- Pdo = AdapterMapDeviceObjectToPdo(Adapter, DeviceObject);
- if (Pdo != NULL) {
- return PdoDispatchPnp(Pdo, DeviceObject, Irp);
+ Target = AdapterMapDeviceObjectToTarget(Adapter, DeviceObject);
+ if (Target != NULL) {
+ return TargetDispatchPnp(Target, DeviceObject, Irp);
}
}
typedef struct _XENVBD_ADAPTER XENVBD_ADAPTER, *PXENVBD_ADAPTER;
#include <storport.h>
-#include "pdo.h"
+#include "target.h"
#include <store_interface.h>
#include <evtchn_interface.h>
#include <gnttab_interface.h>
#include <suspend_interface.h>
#include <unplug_interface.h>
-// Link PDOs
+// Link TARGETs
extern BOOLEAN
-AdapterLinkPdo(
+AdapterLinkTarget(
__in PXENVBD_ADAPTER Adapter,
- __in PXENVBD_PDO Pdo
+ __in PXENVBD_TARGET Target
);
extern BOOLEAN
-AdapterUnlinkPdo(
+AdapterUnlinkTarget(
__in PXENVBD_ADAPTER Adapter,
- __in PXENVBD_PDO Pdo
+ __in PXENVBD_TARGET Target
);
// Query Methods
__checkReturn
#include "blockring.h"
#include "frontend.h"
-#include "pdo.h"
+#include "target.h"
#include "adapter.h"
#include "util.h"
#include "debug.h"
NTSTATUS status;
PCHAR Value;
ULONG Index, RingPages;
- PXENVBD_ADAPTER Adapter = PdoGetAdapter(FrontendGetPdo(BlockRing->Frontend));
+ PXENVBD_ADAPTER Adapter = TargetGetAdapter(FrontendGetTarget(BlockRing->Frontend));
PXENVBD_GRANTER Granter = FrontendGetGranter(BlockRing->Frontend);
ASSERT(BlockRing->Connected == FALSE);
IN PXENVBD_BLOCKRING BlockRing
)
{
- PXENVBD_PDO Pdo = FrontendGetPdo(BlockRing->Frontend);
+ PXENVBD_TARGET Target = FrontendGetTarget(BlockRing->Frontend);
ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
KeAcquireSpinLockAtDpcLevel(&BlockRing->Lock);
if (__BlockRingPutTag(BlockRing, Response->id, &Tag)) {
++BlockRing->Received;
- PdoCompleteResponse(Pdo, Tag, Response->status);
+ TargetCompleteResponse(Target, Tag, Response->status);
}
RtlZeroMemory(Response, sizeof(union blkif_sring_entry));
#include "driver.h"
#include "adapter.h"
-#include "pdo.h"
+#include "target.h"
#include "registry.h"
#include "srbext.h"
#include "buffer.h"
VOID
DriverLinkAdapter(
- __in PXENVBD_ADAPTER Adapter
+ IN PXENVBD_ADAPTER Adapter
)
{
KIRQL Irql;
VOID
DriverUnlinkAdapter(
- __in PXENVBD_ADAPTER Adapter
+ IN PXENVBD_ADAPTER Adapter
)
{
KIRQL Irql;
struct _XENVBD_FRONTEND {
// Frontend
- PXENVBD_PDO Pdo;
+ PXENVBD_TARGET Target;
ULONG TargetId;
ULONG DeviceId;
PCHAR FrontendPath;
{
return Frontend->Inquiry;
}
-PXENVBD_PDO
-FrontendGetPdo(
+PXENVBD_TARGET
+FrontendGetTarget(
__in PXENVBD_FRONTEND Frontend
)
{
- return Frontend->Pdo;
+ return Frontend->Target;
}
PXENVBD_BLOCKRING
FrontendGetBlockRing(
)
{
BlockRingPoll(Frontend->BlockRing);
- PdoSubmitRequests(Frontend->Pdo);
+ TargetSubmitRequests(Frontend->Target);
}
//=============================================================================
Trace("Target[%d] : BackendState(%s) FrontendState(%s)\n",
Frontend->TargetId, XenbusStateName(BackendState), XenbusStateName(FrontendState));
- PdoIssueDeviceEject(Frontend->Pdo, XenbusStateName(BackendState));
+ TargetIssueDeviceEject(Frontend->Target, XenbusStateName(BackendState));
}
}
Verbose("Target[%d] : ===> from %s\n", Frontend->TargetId, __XenvbdStateName(Frontend->State));
State = Frontend->State;
- PdoPreResume(Frontend->Pdo);
+ TargetPreResume(Frontend->Target);
// dont acquire state lock - called at DISPATCH on 1 vCPU with interrupts enabled
Status = __FrontendSetState(Frontend, XENVBD_CLOSED);
ASSERT(FALSE);
}
- PdoPostResume(Frontend->Pdo);
+ TargetPostResume(Frontend->Target);
NotifierTrigger(Frontend->Notifier);
Verbose("Target[%d] : <=== restored %s\n", Frontend->TargetId, __XenvbdStateName(Frontend->State));
KeAcquireSpinLock(&Frontend->StateLock, &Irql);
// acquire interfaces
- Frontend->Store = AdapterAcquireStore(PdoGetAdapter(Frontend->Pdo));
+ Frontend->Store = AdapterAcquireStore(TargetGetAdapter(Frontend->Target));
Status = STATUS_UNSUCCESSFUL;
if (Frontend->Store == NULL)
goto fail1;
- Frontend->Suspend = AdapterAcquireSuspend(PdoGetAdapter(Frontend->Pdo));
+ Frontend->Suspend = AdapterAcquireSuspend(TargetGetAdapter(Frontend->Target));
Status = STATUS_UNSUCCESSFUL;
if (Frontend->Suspend == NULL)
__checkReturn
NTSTATUS
FrontendCreate(
- __in PXENVBD_PDO Pdo,
+ __in PXENVBD_TARGET Target,
__in PCHAR DeviceId,
__in ULONG TargetId,
__out PXENVBD_FRONTEND* _Frontend
goto fail1;
// populate members
- Frontend->Pdo = Pdo;
+ Frontend->Target = Target;
Frontend->TargetId = TargetId;
Frontend->DeviceId = strtoul(DeviceId, NULL, 10);
Frontend->State = XENVBD_INITIALIZED;
Frontend->BackendId = DOMID_INVALID;
Status = STATUS_INSUFFICIENT_RESOURCES;
- Frontend->FrontendPath = DriverFormat("device/%s/%s", AdapterEnum(PdoGetAdapter(Pdo)), DeviceId);
+ Frontend->FrontendPath = DriverFormat("device/%s/%s", AdapterEnum(TargetGetAdapter(Target)), DeviceId);
if (Frontend->FrontendPath == NULL)
goto fail2;
#ifndef _XENVBD_FRONTEND_H
#define _XENVBD_FRONTEND_H
-#include "pdo.h"
+#include "target.h"
#include <debug_interface.h>
typedef enum _XENVBD_STATE {
FrontendGetInquiry(
__in PXENVBD_FRONTEND Frontend
);
-extern PXENVBD_PDO
-FrontendGetPdo(
+extern PXENVBD_TARGET
+FrontendGetTarget(
__in PXENVBD_FRONTEND Frontend
);
#include "blockring.h"
__checkReturn
extern NTSTATUS
FrontendCreate(
- __in PXENVBD_PDO Pdo,
+ __in PXENVBD_TARGET Target,
__in PCHAR DeviceId,
__in ULONG TargetId,
__out PXENVBD_FRONTEND* _Frontend
*/
#include "frontend.h"
-#include "pdo.h"
+#include "target.h"
#include "adapter.h"
#include "util.h"
#include "debug.h"
IN USHORT BackendDomain
)
{
- PXENVBD_ADAPTER Adapter = PdoGetAdapter(FrontendGetPdo(Granter->Frontend));
+ PXENVBD_ADAPTER Adapter = TargetGetAdapter(FrontendGetTarget(Granter->Frontend));
CHAR Name[MAXNAMELEN];
NTSTATUS status;
#include "notifier.h"
#include "frontend.h"
-#include "pdo.h"
+#include "target.h"
#include "adapter.h"
#include "util.h"
#include "debug.h"
IN USHORT BackendDomain
)
{
- PXENVBD_ADAPTER Adapter = PdoGetAdapter(FrontendGetPdo(Notifier->Frontend));
+ PXENVBD_ADAPTER Adapter = TargetGetAdapter(FrontendGetTarget(Notifier->Frontend));
NTSTATUS status;
ASSERT(Notifier->Connected == FALSE);
+++ /dev/null
-/* Copyright (c) Citrix Systems Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms,
- * with or without modification, are permitted provided
- * that the following conditions are met:
- *
- * * Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the
- * following disclaimer.
- * * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the
- * following disclaimer in the documentation and/or other
- * materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "pdo.h"
-#include "driver.h"
-#include "adapter.h"
-#include "frontend.h"
-#include "queue.h"
-#include "srbext.h"
-#include "buffer.h"
-#include "pdoinquiry.h"
-#include "debug.h"
-#include "assert.h"
-#include "util.h"
-#include <xencdb.h>
-#include <names.h>
-#include <store_interface.h>
-#include <evtchn_interface.h>
-#include <gnttab_interface.h>
-#include <debug_interface.h>
-#include <suspend_interface.h>
-
-typedef struct _XENVBD_SG_LIST {
- // SGList from SRB
- PSTOR_SCATTER_GATHER_LIST SGList;
- // "current" values
- STOR_PHYSICAL_ADDRESS PhysAddr;
- ULONG PhysLen;
- // iteration
- ULONG Index;
- ULONG Offset;
- ULONG Length;
-} XENVBD_SG_LIST, *PXENVBD_SG_LIST;
-
-#define PDO_SIGNATURE 'odpX'
-
-typedef struct _XENVBD_LOOKASIDE {
- KEVENT Empty;
- LONG Used;
- LONG Max;
- ULONG Failed;
- ULONG Size;
- NPAGED_LOOKASIDE_LIST List;
-} XENVBD_LOOKASIDE, *PXENVBD_LOOKASIDE;
-
-struct _XENVBD_PDO {
- ULONG Signature;
- PXENVBD_ADAPTER Adapter;
- PDEVICE_OBJECT DeviceObject;
- KEVENT RemoveEvent;
- LONG ReferenceCount;
- DEVICE_PNP_STATE DevicePnpState;
- DEVICE_PNP_STATE PrevPnpState;
- DEVICE_POWER_STATE DevicePowerState;
- KSPIN_LOCK Lock;
-
- // Frontend (Ring, includes XenBus interfaces)
- PXENVBD_FRONTEND Frontend;
- XENVBD_DEVICE_TYPE DeviceType;
-
- // State
- LONG Paused;
-
- // Eject
- BOOLEAN WrittenEjected;
- BOOLEAN EjectRequested;
- BOOLEAN EjectPending;
- BOOLEAN Missing;
- const CHAR* Reason;
-
- // SRBs
- XENVBD_LOOKASIDE RequestList;
- XENVBD_LOOKASIDE SegmentList;
- XENVBD_LOOKASIDE IndirectList;
- XENVBD_QUEUE FreshSrbs;
- XENVBD_QUEUE PreparedReqs;
- XENVBD_QUEUE SubmittedReqs;
- XENVBD_QUEUE ShutdownSrbs;
- ULONG NextTag;
-
- // Stats - SRB Counts by BLKIF_OP_
- ULONG BlkOpRead;
- ULONG BlkOpWrite;
- ULONG BlkOpIndirectRead;
- ULONG BlkOpIndirectWrite;
- ULONG BlkOpBarrier;
- ULONG BlkOpDiscard;
- ULONG BlkOpFlush;
- // Stats - Failures
- ULONG FailedMaps;
- ULONG FailedBounces;
- ULONG FailedGrants;
- // Stats - Segments
- ULONG64 SegsGranted;
- ULONG64 SegsBounced;
-};
-
-//=============================================================================
-#define PDO_POOL_TAG 'odPX'
-#define REQUEST_POOL_TAG 'qeRX'
-#define SEGMENT_POOL_TAG 'geSX'
-#define INDIRECT_POOL_TAG 'dnIX'
-
-__checkReturn
-__drv_allocatesMem(mem)
-__bcount(Size)
-static FORCEINLINE PVOID
-#pragma warning(suppress: 28195)
-__PdoAlloc(
- __in ULONG Size
- )
-{
- return __AllocatePoolWithTag(NonPagedPool, Size, PDO_POOL_TAG);
-}
-
-static FORCEINLINE VOID
-#pragma warning(suppress: 28197)
-__PdoFree(
- __in __drv_freesMem(mem) PVOID Buffer
- )
-{
- if (Buffer)
- __FreePoolWithTag(Buffer, PDO_POOL_TAG);
-}
-
-//=============================================================================
-// Lookasides
-static FORCEINLINE VOID
-__LookasideInit(
- IN OUT PXENVBD_LOOKASIDE Lookaside,
- IN ULONG Size,
- IN ULONG Tag
- )
-{
- RtlZeroMemory(Lookaside, sizeof(XENVBD_LOOKASIDE));
- Lookaside->Size = Size;
- KeInitializeEvent(&Lookaside->Empty, SynchronizationEvent, TRUE);
- ExInitializeNPagedLookasideList(&Lookaside->List, NULL, NULL, 0,
- Size, Tag, 0);
-}
-
-static FORCEINLINE VOID
-__LookasideTerm(
- IN PXENVBD_LOOKASIDE Lookaside
- )
-{
- ASSERT3U(Lookaside->Used, ==, 0);
- ExDeleteNPagedLookasideList(&Lookaside->List);
- RtlZeroMemory(Lookaside, sizeof(XENVBD_LOOKASIDE));
-}
-
-static FORCEINLINE PVOID
-__LookasideAlloc(
- IN PXENVBD_LOOKASIDE Lookaside
- )
-{
- LONG Result;
- PVOID Buffer;
-
- Buffer = ExAllocateFromNPagedLookasideList(&Lookaside->List);
- if (Buffer == NULL) {
- ++Lookaside->Failed;
- return NULL;
- }
-
- RtlZeroMemory(Buffer, Lookaside->Size);
- Result = InterlockedIncrement(&Lookaside->Used);
- ASSERT3S(Result, >, 0);
- if (Result > Lookaside->Max)
- Lookaside->Max = Result;
- KeClearEvent(&Lookaside->Empty);
-
- return Buffer;
-}
-
-static FORCEINLINE VOID
-__LookasideFree(
- IN PXENVBD_LOOKASIDE Lookaside,
- IN PVOID Buffer
- )
-{
- LONG Result;
-
- ExFreeToNPagedLookasideList(&Lookaside->List, Buffer);
- Result = InterlockedDecrement(&Lookaside->Used);
- ASSERT3S(Result, >=, 0);
-
- if (Result == 0) {
- KeSetEvent(&Lookaside->Empty, IO_NO_INCREMENT, FALSE);
- }
-}
-
-static FORCEINLINE VOID
-__LookasideDebug(
- IN PXENVBD_LOOKASIDE Lookaside,
- IN PXENBUS_DEBUG_INTERFACE Debug,
- IN PCHAR Name
- )
-{
- XENBUS_DEBUG(Printf, Debug,
- "LOOKASIDE: %s: %u / %u (%u failed)\n",
- Name, Lookaside->Used,
- Lookaside->Max, Lookaside->Failed);
-
- Lookaside->Max = Lookaside->Used;
- Lookaside->Failed = 0;
-}
-
-//=============================================================================
-// Debug
-static FORCEINLINE PCHAR
-__PnpStateName(
- __in DEVICE_PNP_STATE State
- )
-{
- switch (State) {
- case Invalid: return "Invalid";
- case Present: return "Present";
- case Enumerated: return "Enumerated";
- case Added: return "Added";
- case Started: return "Started";
- case StopPending: return "StopPending";
- case Stopped: return "Stopped";
- case RemovePending: return "RemovePending";
- case SurpriseRemovePending: return "SurpriseRemovePending";
- case Deleted: return "Deleted";
- default: return "UNKNOWN";
- }
-}
-
-DECLSPEC_NOINLINE VOID
-PdoDebugCallback(
- __in PXENVBD_PDO Pdo,
- __in PXENBUS_DEBUG_INTERFACE DebugInterface
- )
-{
- if (Pdo == NULL || DebugInterface == NULL)
- return;
- if (Pdo->Signature != PDO_SIGNATURE)
- return;
-
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: Adapter 0x%p DeviceObject 0x%p\n",
- Pdo->Adapter,
- Pdo->DeviceObject);
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: ReferenceCount %d\n",
- Pdo->ReferenceCount);
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: DevicePnpState %s (%s)\n",
- __PnpStateName(Pdo->DevicePnpState),
- __PnpStateName(Pdo->PrevPnpState));
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: DevicePowerState %s\n",
- PowerDeviceStateName(Pdo->DevicePowerState));
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: %s\n",
- Pdo->Missing ? Pdo->Reason : "Not Missing");
-
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: BLKIF_OPs: READ=%u WRITE=%u\n",
- Pdo->BlkOpRead, Pdo->BlkOpWrite);
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n",
- Pdo->BlkOpIndirectRead, Pdo->BlkOpIndirectWrite);
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n",
- Pdo->BlkOpBarrier, Pdo->BlkOpDiscard, Pdo->BlkOpFlush);
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: Failed: Maps=%u Bounces=%u Grants=%u\n",
- Pdo->FailedMaps, Pdo->FailedBounces, Pdo->FailedGrants);
- XENBUS_DEBUG(Printf, DebugInterface,
- "PDO: Segments Granted=%llu Bounced=%llu\n",
- Pdo->SegsGranted, Pdo->SegsBounced);
-
- __LookasideDebug(&Pdo->RequestList, DebugInterface, "REQUESTs");
- __LookasideDebug(&Pdo->SegmentList, DebugInterface, "SEGMENTs");
- __LookasideDebug(&Pdo->IndirectList, DebugInterface, "INDIRECTs");
-
- QueueDebugCallback(&Pdo->FreshSrbs, "Fresh ", DebugInterface);
- QueueDebugCallback(&Pdo->PreparedReqs, "Prepared ", DebugInterface);
- QueueDebugCallback(&Pdo->SubmittedReqs, "Submitted", DebugInterface);
- QueueDebugCallback(&Pdo->ShutdownSrbs, "Shutdown ", DebugInterface);
-
- FrontendDebugCallback(Pdo->Frontend, DebugInterface);
-
- Pdo->BlkOpRead = Pdo->BlkOpWrite = 0;
- Pdo->BlkOpIndirectRead = Pdo->BlkOpIndirectWrite = 0;
- Pdo->BlkOpBarrier = Pdo->BlkOpDiscard = Pdo->BlkOpFlush = 0;
- Pdo->FailedMaps = Pdo->FailedBounces = Pdo->FailedGrants = 0;
- Pdo->SegsGranted = Pdo->SegsBounced = 0;
-}
-
-//=============================================================================
-// Power States
-__checkReturn
-static FORCEINLINE BOOLEAN
-PdoSetDevicePowerState(
- __in PXENVBD_PDO Pdo,
- __in DEVICE_POWER_STATE State
- )
-{
- KIRQL Irql;
- BOOLEAN Changed = FALSE;
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- if (Pdo->DevicePowerState != State) {
- Verbose("Target[%d] : POWER %s to %s\n", PdoGetTargetId(Pdo), PowerDeviceStateName(Pdo->DevicePowerState), PowerDeviceStateName(State));
- Pdo->DevicePowerState = State;
- Changed = TRUE;
- }
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-
- return Changed;
-}
-
-//=============================================================================
-// PnP States
-FORCEINLINE VOID
-PdoSetMissing(
- __in PXENVBD_PDO Pdo,
- __in __nullterminated const CHAR* Reason
- )
-{
- KIRQL Irql;
-
- ASSERT3P(Reason, !=, NULL);
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- if (Pdo->Missing) {
- Verbose("Target[%d] : Already MISSING (%s) when trying to set (%s)\n", PdoGetTargetId(Pdo), Pdo->Reason, Reason);
- } else {
- Verbose("Target[%d] : MISSING %s\n", PdoGetTargetId(Pdo), Reason);
- Pdo->Missing = TRUE;
- Pdo->Reason = Reason;
- }
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-}
-
-__checkReturn
-FORCEINLINE BOOLEAN
-PdoIsMissing(
- __in PXENVBD_PDO Pdo
- )
-{
- KIRQL Irql;
- BOOLEAN Missing;
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- Missing = Pdo->Missing;
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-
- return Missing;
-}
-
-FORCEINLINE const CHAR*
-PdoMissingReason(
- __in PXENVBD_PDO Pdo
- )
-{
- KIRQL Irql;
- const CHAR* Reason;
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- Reason = Pdo->Reason;
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-
- return Reason;
-}
-
-FORCEINLINE VOID
-PdoSetDevicePnpState(
- __in PXENVBD_PDO Pdo,
- __in DEVICE_PNP_STATE State
- )
-{
- Verbose("Target[%d] : PNP %s to %s\n",
- PdoGetTargetId(Pdo),
- __PnpStateName(Pdo->DevicePnpState),
- __PnpStateName(State));
-
- if (Pdo->DevicePnpState == Deleted)
- return;
-
- Pdo->PrevPnpState = Pdo->DevicePnpState;
- Pdo->DevicePnpState = State;
-}
-
-__checkReturn
-FORCEINLINE DEVICE_PNP_STATE
-PdoGetDevicePnpState(
- __in PXENVBD_PDO Pdo
- )
-{
- return Pdo->DevicePnpState;
-}
-
-static FORCEINLINE VOID
-__PdoRestoreDevicePnpState(
- __in PXENVBD_PDO Pdo,
- __in DEVICE_PNP_STATE State
- )
-{
- if (Pdo->DevicePnpState == State) {
- Verbose("Target[%d] : PNP %s to %s\n", PdoGetTargetId(Pdo), __PnpStateName(Pdo->DevicePnpState), __PnpStateName(Pdo->PrevPnpState));
- Pdo->DevicePnpState = Pdo->PrevPnpState;
- }
-}
-
-//=============================================================================
-// Reference Counting
-FORCEINLINE LONG
-__PdoReference(
- __in PXENVBD_PDO Pdo,
- __in PCHAR Caller
- )
-{
- LONG Result;
-
- ASSERT3P(Pdo, !=, NULL);
- Result = InterlockedIncrement(&Pdo->ReferenceCount);
- ASSERTREFCOUNT(Result, >, 0, Caller);
-
- if (Result == 1) {
- Result = InterlockedDecrement(&Pdo->ReferenceCount);
- Error("Target[%d] : %s: Attempting to take reference of removed PDO from %d\n", PdoGetTargetId(Pdo), Caller, Result);
- return 0;
- } else {
- ASSERTREFCOUNT(Result, >, 1, Caller);
- return Result;
- }
-}
-
-FORCEINLINE LONG
-__PdoDereference(
- __in PXENVBD_PDO Pdo,
- __in PCHAR Caller
- )
-{
- LONG Result;
-
- ASSERT3P(Pdo, !=, NULL);
- Result = InterlockedDecrement(&Pdo->ReferenceCount);
- ASSERTREFCOUNT(Result, >=, 0, Caller);
-
- if (Result == 0) {
- Verbose("Final ReferenceCount dropped, Target[%d] able to be removed\n", PdoGetTargetId(Pdo));
- KeSetEvent(&Pdo->RemoveEvent, IO_NO_INCREMENT, FALSE);
- }
- return Result;
-}
-
-//=============================================================================
-// Query Methods
-FORCEINLINE ULONG
-PdoGetTargetId(
- __in PXENVBD_PDO Pdo
- )
-{
- ASSERT3P(Pdo, !=, NULL);
- return FrontendGetTargetId(Pdo->Frontend);
-}
-
-__checkReturn
-FORCEINLINE PDEVICE_OBJECT
-PdoGetDeviceObject(
- __in PXENVBD_PDO Pdo
- )
-{
- ASSERT3P(Pdo, !=, NULL);
- return Pdo->DeviceObject;
-}
-
-FORCEINLINE VOID
-PdoSetDeviceObject(
- __in PXENVBD_PDO Pdo,
- __in PDEVICE_OBJECT DeviceObject
- )
-{
- Verbose("Target[%d] : Setting DeviceObject = 0x%p\n", PdoGetTargetId(Pdo), DeviceObject);
-
- ASSERT3P(Pdo->DeviceObject, ==, NULL);
- Pdo->DeviceObject = DeviceObject;
-}
-
-__checkReturn
-FORCEINLINE BOOLEAN
-PdoIsPaused(
- __in PXENVBD_PDO Pdo
- )
-{
- BOOLEAN Paused;
- KIRQL Irql;
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- Paused = (Pdo->Paused > 0);
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-
- return Paused;
-}
-
-__checkReturn
-FORCEINLINE ULONG
-PdoOutstandingReqs(
- __in PXENVBD_PDO Pdo
- )
-{
- return QueueCount(&Pdo->SubmittedReqs);
-}
-
-__checkReturn
-FORCEINLINE PXENVBD_ADAPTER
-PdoGetAdapter(
- __in PXENVBD_PDO Pdo
- )
-{
- return Pdo->Adapter;
-}
-
-FORCEINLINE ULONG
-PdoSectorSize(
- __in PXENVBD_PDO Pdo
- )
-{
- return FrontendGetDiskInfo(Pdo->Frontend)->SectorSize;
-}
-
-//=============================================================================
-static PXENVBD_INDIRECT
-PdoGetIndirect(
- IN PXENVBD_PDO Pdo
- )
-{
- PXENVBD_INDIRECT Indirect;
- NTSTATUS status;
- PXENVBD_GRANTER Granter = FrontendGetGranter(Pdo->Frontend);
-
- Indirect = __LookasideAlloc(&Pdo->IndirectList);
- if (Indirect == NULL)
- goto fail1;
-
- RtlZeroMemory(Indirect, sizeof(XENVBD_INDIRECT));
-
- Indirect->Mdl = __AllocatePage();
- if (Indirect->Mdl == NULL)
- goto fail2;
-
- Indirect->Page = MmGetSystemAddressForMdlSafe(Indirect->Mdl,
- NormalPagePriority);
-
- status = GranterGet(Granter,
- MmGetMdlPfnArray(Indirect->Mdl)[0],
- TRUE,
- &Indirect->Grant);
- if (!NT_SUCCESS(status))
- goto fail3;
-
- return Indirect;
-
-fail3:
- __FreePage(Indirect->Mdl);
-fail2:
- __LookasideFree(&Pdo->IndirectList, Indirect);
-fail1:
- return NULL;
-}
-
-static VOID
-PdoPutIndirect(
- IN PXENVBD_PDO Pdo,
- IN PXENVBD_INDIRECT Indirect
- )
-{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Pdo->Frontend);
-
- if (Indirect->Grant)
- GranterPut(Granter, Indirect->Grant);
- if (Indirect->Page)
- __FreePage(Indirect->Mdl);
-
- RtlZeroMemory(Indirect, sizeof(XENVBD_INDIRECT));
- __LookasideFree(&Pdo->IndirectList, Indirect);
-}
-
-static PXENVBD_SEGMENT
-PdoGetSegment(
- IN PXENVBD_PDO Pdo
- )
-{
- PXENVBD_SEGMENT Segment;
-
- Segment = __LookasideAlloc(&Pdo->SegmentList);
- if (Segment == NULL)
- goto fail1;
-
- RtlZeroMemory(Segment, sizeof(XENVBD_SEGMENT));
- return Segment;
-
-fail1:
- return NULL;
-}
-
-static VOID
-PdoPutSegment(
- IN PXENVBD_PDO Pdo,
- IN PXENVBD_SEGMENT Segment
- )
-{
- PXENVBD_GRANTER Granter = FrontendGetGranter(Pdo->Frontend);
-
- if (Segment->Grant)
- GranterPut(Granter, Segment->Grant);
-
- if (Segment->BufferId)
- BufferPut(Segment->BufferId);
-
- if (Segment->Buffer)
- MmUnmapLockedPages(Segment->Buffer, &Segment->Mdl);
-
- RtlZeroMemory(Segment, sizeof(XENVBD_SEGMENT));
- __LookasideFree(&Pdo->SegmentList, Segment);
-}
-
-static PXENVBD_REQUEST
-PdoGetRequest(
- IN PXENVBD_PDO Pdo
- )
-{
- PXENVBD_REQUEST Request;
-
- Request = __LookasideAlloc(&Pdo->RequestList);
- if (Request == NULL)
- goto fail1;
-
- RtlZeroMemory(Request, sizeof(XENVBD_REQUEST));
- Request->Id = (ULONG)InterlockedIncrement((PLONG)&Pdo->NextTag);
- InitializeListHead(&Request->Segments);
- InitializeListHead(&Request->Indirects);
-
- return Request;
-
-fail1:
- return NULL;
-}
-
-static VOID
-PdoPutRequest(
- IN PXENVBD_PDO Pdo,
- IN PXENVBD_REQUEST Request
- )
-{
- PLIST_ENTRY Entry;
-
- for (;;) {
- PXENVBD_SEGMENT Segment;
-
- Entry = RemoveHeadList(&Request->Segments);
- if (Entry == &Request->Segments)
- break;
- Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry);
- PdoPutSegment(Pdo, Segment);
- }
-
- for (;;) {
- PXENVBD_INDIRECT Indirect;
-
- Entry = RemoveHeadList(&Request->Indirects);
- if (Entry == &Request->Indirects)
- break;
- Indirect = CONTAINING_RECORD(Entry, XENVBD_INDIRECT, Entry);
- PdoPutIndirect(Pdo, Indirect);
- }
-
- RtlZeroMemory(Request, sizeof(XENVBD_REQUEST));
- __LookasideFree(&Pdo->RequestList, Request);
-}
-
-static FORCEINLINE PXENVBD_REQUEST
-PdoRequestFromTag(
- IN PXENVBD_PDO Pdo,
- IN ULONG Tag
- )
-{
- KIRQL Irql;
- PLIST_ENTRY Entry;
- PXENVBD_QUEUE Queue = &Pdo->SubmittedReqs;
-
- KeAcquireSpinLock(&Queue->Lock, &Irql);
-
- for (Entry = Queue->List.Flink; Entry != &Queue->List; Entry = Entry->Flink) {
- PXENVBD_REQUEST Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
- if (Request->Id == Tag) {
- RemoveEntryList(&Request->Entry);
- --Queue->Current;
- KeReleaseSpinLock(&Queue->Lock, Irql);
- return Request;
- }
- }
-
- KeReleaseSpinLock(&Queue->Lock, Irql);
- Warning("Target[%d] : Tag %x not found in submitted list (%u items)\n",
- PdoGetTargetId(Pdo), Tag, QueueCount(Queue));
- return NULL;
-}
-
-static FORCEINLINE VOID
-__PdoIncBlkifOpCount(
- __in PXENVBD_PDO Pdo,
- __in PXENVBD_REQUEST Request
- )
-{
- switch (Request->Operation) {
- case BLKIF_OP_READ:
- if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
- ++Pdo->BlkOpIndirectRead;
- else
- ++Pdo->BlkOpRead;
- break;
- case BLKIF_OP_WRITE:
- if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
- ++Pdo->BlkOpIndirectWrite;
- else
- ++Pdo->BlkOpWrite;
- break;
- case BLKIF_OP_WRITE_BARRIER:
- ++Pdo->BlkOpBarrier;
- break;
- case BLKIF_OP_DISCARD:
- ++Pdo->BlkOpDiscard;
- break;
- case BLKIF_OP_FLUSH_DISKCACHE:
- ++Pdo->BlkOpFlush;
- break;
- default:
- ASSERT(FALSE);
- break;
- }
-}
-
-static FORCEINLINE ULONG
-__SectorsPerPage(
- __in ULONG SectorSize
- )
-{
- ASSERT3U(SectorSize, !=, 0);
- return PAGE_SIZE / SectorSize;
-}
-
-static FORCEINLINE VOID
-__Operation(
- __in UCHAR CdbOp,
- __out PUCHAR RingOp,
- __out PBOOLEAN ReadOnly
- )
-{
- switch (CdbOp) {
- case SCSIOP_READ:
- *RingOp = BLKIF_OP_READ;
- *ReadOnly = FALSE;
- break;
- case SCSIOP_WRITE:
- *RingOp = BLKIF_OP_WRITE;
- *ReadOnly = TRUE;
- break;
- default:
- ASSERT(FALSE);
- }
-}
-
-static FORCEINLINE ULONG
-__Offset(
- __in STOR_PHYSICAL_ADDRESS PhysAddr
- )
-{
- return (ULONG)(PhysAddr.QuadPart & (PAGE_SIZE - 1));
-}
-
-static FORCEINLINE PFN_NUMBER
-__Phys2Pfn(
- __in STOR_PHYSICAL_ADDRESS PhysAddr
- )
-{
- return (PFN_NUMBER)(PhysAddr.QuadPart >> PAGE_SHIFT);
-}
-
-static FORCEINLINE PFN_NUMBER
-__Virt2Pfn(
- __in PVOID VirtAddr
- )
-{
- return (PFN_NUMBER)(MmGetPhysicalAddress(VirtAddr).QuadPart >> PAGE_SHIFT);
-}
-
-static FORCEINLINE MM_PAGE_PRIORITY
-__PdoPriority(
- __in PXENVBD_PDO Pdo
- )
-{
- PXENVBD_CAPS Caps = FrontendGetCaps(Pdo->Frontend);
- if (!(Caps->Paging ||
- Caps->Hibernation ||
- Caps->DumpFile))
- return NormalPagePriority;
-
- return HighPagePriority;
-}
-
-#define __min(_x, _y) ((_x) < (_y)) ? (_x) : (_y)
-
-static FORCEINLINE VOID
-SGListGet(
- IN OUT PXENVBD_SG_LIST SGList
- )
-{
- PSTOR_SCATTER_GATHER_ELEMENT SGElement;
-
- ASSERT3U(SGList->Index, <, SGList->SGList->NumberOfElements);
-
- SGElement = &SGList->SGList->List[SGList->Index];
-
- SGList->PhysAddr.QuadPart = SGElement->PhysicalAddress.QuadPart + SGList->Offset;
- SGList->PhysLen = __min(PAGE_SIZE - __Offset(SGList->PhysAddr) - SGList->Length, SGElement->Length - SGList->Offset);
-
- ASSERT3U(SGList->PhysLen, <=, PAGE_SIZE);
- ASSERT3U(SGList->Offset, <, SGElement->Length);
-
- SGList->Length = SGList->PhysLen; // gets reset every time for Granted, every 1or2 times for Bounced
- SGList->Offset = SGList->Offset + SGList->PhysLen;
- if (SGList->Offset >= SGElement->Length) {
- SGList->Index = SGList->Index + 1;
- SGList->Offset = 0;
- }
-}
-
-static FORCEINLINE BOOLEAN
-SGListNext(
- IN OUT PXENVBD_SG_LIST SGList,
- IN ULONG AlignmentMask
- )
-{
- SGList->Length = 0;
- SGListGet(SGList); // get next PhysAddr and PhysLen
- return !((SGList->PhysAddr.QuadPart & AlignmentMask) || (SGList->PhysLen & AlignmentMask));
-}
-
-static FORCEINLINE BOOLEAN
-MapSegmentBuffer(
- IN PXENVBD_PDO Pdo,
- IN PXENVBD_SEGMENT Segment,
- IN PXENVBD_SG_LIST SGList,
- IN ULONG SectorSize,
- IN ULONG SectorsNow
- )
-{
- PMDL Mdl;
-
- // map PhysAddr to 1 or 2 pages and lock for VirtAddr
-#pragma warning(push)
-#pragma warning(disable:28145)
- Mdl = &Segment->Mdl;
- Mdl->Next = NULL;
- Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER));
- Mdl->MdlFlags = MDL_PAGES_LOCKED;
- Mdl->Process = NULL;
- Mdl->MappedSystemVa = NULL;
- Mdl->StartVa = NULL;
- Mdl->ByteCount = SGList->PhysLen;
- Mdl->ByteOffset = __Offset(SGList->PhysAddr);
- Segment->Pfn[0] = __Phys2Pfn(SGList->PhysAddr);
-
- if (SGList->PhysLen < SectorsNow * SectorSize) {
- SGListGet(SGList);
- Mdl->Size += sizeof(PFN_NUMBER);
- Mdl->ByteCount = Mdl->ByteCount + SGList->PhysLen;
- Segment->Pfn[1] = __Phys2Pfn(SGList->PhysAddr);
- }
-#pragma warning(pop)
-
- ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0);
- ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE);
- ASSERT3U(SectorsNow, ==, (Mdl->ByteCount / SectorSize));
-
- Segment->Length = __min(Mdl->ByteCount, PAGE_SIZE);
- Segment->Buffer = MmMapLockedPagesSpecifyCache(Mdl, KernelMode,
- MmCached, NULL, FALSE, __PdoPriority(Pdo));
- if (!Segment->Buffer) {
- goto fail;
- }
-
- ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Segment->Pfn[0]);
- ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Segment->Pfn[1]);
-
- return TRUE;
-
-fail:
- return FALSE;
-}
-
-static FORCEINLINE VOID
-RequestCopyOutput(
- __in PXENVBD_REQUEST Request
- )
-{
- PLIST_ENTRY Entry;
-
- if (Request->Operation != BLKIF_OP_READ)
- return;
-
- for (Entry = Request->Segments.Flink;
- Entry != &Request->Segments;
- Entry = Entry->Flink) {
- PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry);
-
- if (Segment->BufferId)
- BufferCopyOut(Segment->BufferId, Segment->Buffer, Segment->Length);
- }
-}
-
-static BOOLEAN
-PrepareSegment(
- IN PXENVBD_PDO Pdo,
- IN PXENVBD_SEGMENT Segment,
- IN PXENVBD_SG_LIST SGList,
- IN BOOLEAN ReadOnly,
- IN ULONG SectorsLeft,
- OUT PULONG SectorsNow
- )
-{
- PFN_NUMBER Pfn;
- NTSTATUS Status;
- PXENVBD_GRANTER Granter = FrontendGetGranter(Pdo->Frontend);
- const ULONG SectorSize = PdoSectorSize(Pdo);
- const ULONG SectorsPerPage = __SectorsPerPage(SectorSize);
-
- if (SGListNext(SGList, SectorSize - 1)) {
- ++Pdo->SegsGranted;
- // get first sector, last sector and count
- Segment->FirstSector = (UCHAR)((__Offset(SGList->PhysAddr) + SectorSize - 1) / SectorSize);
- *SectorsNow = __min(SectorsLeft, SectorsPerPage - Segment->FirstSector);
- Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow - 1);
- Segment->BufferId = NULL; // granted, ensure its null
- Segment->Buffer = NULL; // granted, ensure its null
- Segment->Length = 0; // granted, ensure its 0
- Pfn = __Phys2Pfn(SGList->PhysAddr);
-
- ASSERT3U((SGList->PhysLen / SectorSize), ==, *SectorsNow);
- ASSERT3U((SGList->PhysLen & (SectorSize - 1)), ==, 0);
- } else {
- ++Pdo->SegsBounced;
- // get first sector, last sector and count
- Segment->FirstSector = 0;
- *SectorsNow = __min(SectorsLeft, SectorsPerPage);
- Segment->LastSector = (UCHAR)(*SectorsNow - 1);
-
- // map SGList to Virtual Address. Populates Segment->Buffer and Segment->Length
- if (!MapSegmentBuffer(Pdo, Segment, SGList, SectorSize, *SectorsNow)) {
- ++Pdo->FailedMaps;
- goto fail1;
- }
-
- // get a buffer
- if (!BufferGet(Segment, &Segment->BufferId, &Pfn)) {
- ++Pdo->FailedBounces;
- goto fail2;
- }
-
- // copy contents in
- if (ReadOnly) { // Operation == BLKIF_OP_WRITE
- BufferCopyIn(Segment->BufferId, Segment->Buffer, Segment->Length);
- }
- }
-
- // Grant segment's page
- Status = GranterGet(Granter, Pfn, ReadOnly, &Segment->Grant);
- if (!NT_SUCCESS(Status)) {
- ++Pdo->FailedGrants;
- goto fail3;
- }
-
- return TRUE;
-
-fail3:
-fail2:
-fail1:
- return FALSE;
-}
-
-static BOOLEAN
-PrepareBlkifReadWrite(
- IN PXENVBD_PDO Pdo,
- IN PXENVBD_REQUEST Request,
- IN PXENVBD_SG_LIST SGList,
- IN ULONG MaxSegments,
- IN ULONG64 SectorStart,
- IN ULONG SectorsLeft,
- OUT PULONG SectorsDone
- )
-{
- UCHAR Operation;
- BOOLEAN ReadOnly;
- ULONG Index;
- __Operation(Cdb_OperationEx(Request->Srb), &Operation, &ReadOnly);
-
- Request->Operation = Operation;
- Request->NrSegments = 0;
- Request->FirstSector = SectorStart;
-
- for (Index = 0;
- Index < MaxSegments &&
- SectorsLeft > 0;
- ++Index) {
- PXENVBD_SEGMENT Segment;
- ULONG SectorsNow;
-
- Segment = PdoGetSegment(Pdo);
- if (Segment == NULL)
- goto fail1;
-
- InsertTailList(&Request->Segments, &Segment->Entry);
- ++Request->NrSegments;
-
- if (!PrepareSegment(Pdo,
- Segment,
- SGList,
- ReadOnly,
- SectorsLeft,
- &SectorsNow))
- goto fail2;
-
- *SectorsDone += SectorsNow;
- SectorsLeft -= SectorsNow;
- }
- ASSERT3U(Request->NrSegments, >, 0);
- ASSERT3U(Request->NrSegments, <=, MaxSegments);
-
- return TRUE;
-
-fail2:
-fail1:
- return FALSE;
-}
-
-static BOOLEAN
-PrepareBlkifIndirect(
- IN PXENVBD_PDO Pdo,
- IN PXENVBD_REQUEST Request
- )
-{
- ULONG Index;
- ULONG NrSegments = 0;
-
- for (Index = 0;
- Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
- NrSegments < Request->NrSegments;
- ++Index) {
- PXENVBD_INDIRECT Indirect;
-
- Indirect = PdoGetIndirect(Pdo);
- if (Indirect == NULL)
- goto fail1;
- InsertTailList(&Request->Indirects, &Indirect->Entry);
-
- NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE;
- }
-
- return TRUE;
-
-fail1:
- return FALSE;
-}
-
-static FORCEINLINE ULONG
-UseIndirect(
- IN PXENVBD_PDO Pdo,
- IN ULONG SectorsLeft
- )
-{
- const ULONG SectorsPerPage = __SectorsPerPage(PdoSectorSize(Pdo));
- const ULONG MaxIndirectSegs = FrontendGetFeatures(Pdo->Frontend)->Indirect;
-
- if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
- return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported
-
- if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage)
- return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single BLKIF_OP_{READ/WRITE}
-
- return MaxIndirectSegs;
-}
-
-static FORCEINLINE ULONG
-PdoQueueRequestList(
- IN PXENVBD_PDO Pdo,
- IN PLIST_ENTRY List
- )
-{
- ULONG Count = 0;
- for (;;) {
- PXENVBD_REQUEST Request;
- PLIST_ENTRY Entry;
-
- Entry = RemoveHeadList(List);
- if (Entry == List)
- break;
-
- ++Count;
- Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
- __PdoIncBlkifOpCount(Pdo, Request);
- QueueAppend(&Pdo->PreparedReqs, &Request->Entry);
- }
- return Count;
-}
-
-static FORCEINLINE VOID
-PdoCancelRequestList(
- IN PXENVBD_PDO Pdo,
- IN PLIST_ENTRY List
- )
-{
- for (;;) {
- PXENVBD_REQUEST Request;
- PLIST_ENTRY Entry;
-
- Entry = RemoveHeadList(List);
- if (Entry == List)
- break;
-
- Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
- PdoPutRequest(Pdo, Request);
- }
-}
-
-__checkReturn
-static BOOLEAN
-PrepareReadWrite(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
- ULONG64 SectorStart = Cdb_LogicalBlock(Srb);
- ULONG SectorsLeft = Cdb_TransferBlock(Srb);
- LIST_ENTRY List;
- XENVBD_SG_LIST SGList;
- ULONG DebugCount;
-
- Srb->SrbStatus = SRB_STATUS_PENDING;
-
- InitializeListHead(&List);
- SrbExt->Count = 0;
-
- RtlZeroMemory(&SGList, sizeof(SGList));
- SGList.SGList = StorPortGetScatterGatherList(PdoGetAdapter(Pdo), Srb);
-
- while (SectorsLeft > 0) {
- ULONG MaxSegments;
- ULONG SectorsDone = 0;
- PXENVBD_REQUEST Request;
-
- Request = PdoGetRequest(Pdo);
- if (Request == NULL)
- goto fail1;
- InsertTailList(&List, &Request->Entry);
- InterlockedIncrement(&SrbExt->Count);
-
- Request->Srb = Srb;
- MaxSegments = UseIndirect(Pdo, SectorsLeft);
-
- if (!PrepareBlkifReadWrite(Pdo,
- Request,
- &SGList,
- MaxSegments,
- SectorStart,
- SectorsLeft,
- &SectorsDone))
- goto fail2;
-
- if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
- if (!PrepareBlkifIndirect(Pdo, Request))
- goto fail3;
- }
-
- SectorsLeft -= SectorsDone;
- SectorStart += SectorsDone;
- }
-
- DebugCount = PdoQueueRequestList(Pdo, &List);
- if (DebugCount != (ULONG)SrbExt->Count) {
- Trace("[%u] %d != %u\n", PdoGetTargetId(Pdo), SrbExt->Count, DebugCount);
- }
- return TRUE;
-
-fail3:
-fail2:
-fail1:
- PdoCancelRequestList(Pdo, &List);
- SrbExt->Count = 0;
- Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
-}
-
-__checkReturn
-static BOOLEAN
-PrepareSyncCache(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
- PXENVBD_REQUEST Request;
- LIST_ENTRY List;
- UCHAR Operation;
- ULONG DebugCount;
-
- Srb->SrbStatus = SRB_STATUS_PENDING;
-
- if (FrontendGetDiskInfo(Pdo->Frontend)->FlushCache)
- Operation = BLKIF_OP_FLUSH_DISKCACHE;
- else
- Operation = BLKIF_OP_WRITE_BARRIER;
-
- InitializeListHead(&List);
- SrbExt->Count = 0;
-
- Request = PdoGetRequest(Pdo);
- if (Request == NULL)
- goto fail1;
- InsertTailList(&List, &Request->Entry);
- InterlockedIncrement(&SrbExt->Count);
-
- Request->Srb = Srb;
- Request->Operation = Operation;
- Request->FirstSector = Cdb_LogicalBlock(Srb);
-
- DebugCount = PdoQueueRequestList(Pdo, &List);
- if (DebugCount != (ULONG)SrbExt->Count) {
- Trace("[%u] %d != %u\n", PdoGetTargetId(Pdo), SrbExt->Count, DebugCount);
- }
- return TRUE;
-
-fail1:
- PdoCancelRequestList(Pdo, &List);
- SrbExt->Count = 0;
- Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
-}
-
-__checkReturn
-static BOOLEAN
-PrepareUnmap(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
- PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer;
- ULONG Count = _byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) / sizeof(UNMAP_BLOCK_DESCRIPTOR);
- ULONG Index;
- LIST_ENTRY List;
- ULONG DebugCount;
-
- Srb->SrbStatus = SRB_STATUS_PENDING;
-
- InitializeListHead(&List);
- SrbExt->Count = 0;
-
- for (Index = 0; Index < Count; ++Index) {
- PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index];
- PXENVBD_REQUEST Request;
-
- Request = PdoGetRequest(Pdo);
- if (Request == NULL)
- goto fail1;
- InsertTailList(&List, &Request->Entry);
- InterlockedIncrement(&SrbExt->Count);
-
- Request->Srb = Srb;
- Request->Operation = BLKIF_OP_DISCARD;
- Request->FirstSector = _byteswap_uint64(*(PULONG64)Descr->StartingLba);
- Request->NrSectors = _byteswap_ulong(*(PULONG)Descr->LbaCount);
- Request->Flags = 0;
- }
-
- DebugCount = PdoQueueRequestList(Pdo, &List);
- if (DebugCount != (ULONG)SrbExt->Count) {
- Trace("[%u] %d != %u\n", PdoGetTargetId(Pdo), SrbExt->Count, DebugCount);
- }
- return TRUE;
-
-fail1:
- PdoCancelRequestList(Pdo, &List);
- SrbExt->Count = 0;
- Srb->SrbStatus = SRB_STATUS_ERROR;
- return FALSE;
-}
-
-//=============================================================================
-// Queue-Related
-static FORCEINLINE VOID
-__PdoPauseDataPath(
- __in PXENVBD_PDO Pdo,
- __in BOOLEAN Timeout
- )
-{
- KIRQL Irql;
- ULONG Requests;
- ULONG Count = 0;
- PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Pdo->Frontend);
- PXENVBD_BLOCKRING BlockRing = FrontendGetBlockRing(Pdo->Frontend);
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- ++Pdo->Paused;
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-
- Requests = QueueCount(&Pdo->SubmittedReqs);
- KeMemoryBarrier();
-
- Verbose("Target[%d] : Waiting for %d Submitted requests\n", PdoGetTargetId(Pdo), Requests);
-
- // poll ring and send event channel notification every 1ms (for up to 3 minutes)
- while (QueueCount(&Pdo->SubmittedReqs)) {
- if (Timeout && Count > 180000)
- break;
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
- BlockRingPoll(BlockRing);
- KeLowerIrql(Irql);
- NotifierSend(Notifier); // let backend know it needs to do some work
- StorPortStallExecution(1000); // 1000 micro-seconds
- ++Count;
- }
-
- Verbose("Target[%d] : %u/%u Submitted requests left (%u iterrations)\n",
- PdoGetTargetId(Pdo), QueueCount(&Pdo->SubmittedReqs), Requests, Count);
-
- // Abort Fresh SRBs
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PLIST_ENTRY Entry = QueuePop(&Pdo->FreshSrbs);
- if (Entry == NULL)
- break;
- SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
-
- Verbose("Target[%d] : FreshSrb 0x%p -> SCSI_ABORTED\n", PdoGetTargetId(Pdo), SrbExt->Srb);
- SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
- SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED;
- AdapterCompleteSrb(PdoGetAdapter(Pdo), SrbExt->Srb);
- }
-
- // Fail PreparedReqs
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PXENVBD_REQUEST Request;
- PLIST_ENTRY Entry = QueuePop(&Pdo->PreparedReqs);
- if (Entry == NULL)
- break;
- Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
- SrbExt = GetSrbExt(Request->Srb);
-
- Verbose("Target[%d] : PreparedReq 0x%p -> FAILED\n", PdoGetTargetId(Pdo), Request);
-
- SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
- PdoPutRequest(Pdo, Request);
-
- if (InterlockedDecrement(&SrbExt->Count) == 0) {
- SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED
- AdapterCompleteSrb(PdoGetAdapter(Pdo), SrbExt->Srb);
- }
- }
-}
-
-static FORCEINLINE VOID
-__PdoUnpauseDataPath(
- __in PXENVBD_PDO Pdo
- )
-{
- KIRQL Irql;
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- --Pdo->Paused;
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-}
-
-static FORCEINLINE BOOLEAN
-PdoPrepareFresh(
- IN PXENVBD_PDO Pdo
- )
-{
- PXENVBD_SRBEXT SrbExt;
- PLIST_ENTRY Entry;
-
- Entry = QueuePop(&Pdo->FreshSrbs);
- if (Entry == NULL)
- return FALSE; // fresh queue is empty
-
- SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
-
- switch (Cdb_OperationEx(SrbExt->Srb)) {
- case SCSIOP_READ:
- case SCSIOP_WRITE:
- if (PrepareReadWrite(Pdo, SrbExt->Srb))
- return TRUE; // prepared this SRB
- break;
- case SCSIOP_SYNCHRONIZE_CACHE:
- if (PrepareSyncCache(Pdo, SrbExt->Srb))
- return TRUE; // prepared this SRB
- break;
- case SCSIOP_UNMAP:
- if (PrepareUnmap(Pdo, SrbExt->Srb))
- return TRUE; // prepared this SRB
- break;
- default:
- ASSERT(FALSE);
- break;
- }
- QueueUnPop(&Pdo->FreshSrbs, &SrbExt->Entry);
-
- return FALSE; // prepare failed
-}
-
-static FORCEINLINE BOOLEAN
-PdoSubmitPrepared(
- __in PXENVBD_PDO Pdo
- )
-{
- PXENVBD_BLOCKRING BlockRing = FrontendGetBlockRing(Pdo->Frontend);
- if (PdoIsPaused(Pdo)) {
- if (QueueCount(&Pdo->PreparedReqs))
- Warning("Target[%d] : Paused, not submitting new requests (%u)\n",
- PdoGetTargetId(Pdo),
- QueueCount(&Pdo->PreparedReqs));
- return FALSE;
- }
-
- for (;;) {
- PXENVBD_REQUEST Request;
- PLIST_ENTRY Entry;
-
- Entry = QueuePop(&Pdo->PreparedReqs);
- if (Entry == NULL)
- break;
-
- Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
-
- QueueAppend(&Pdo->SubmittedReqs, &Request->Entry);
- KeMemoryBarrier();
-
- if (BlockRingSubmit(BlockRing, Request))
- continue;
-
- QueueRemove(&Pdo->SubmittedReqs, &Request->Entry);
- QueueUnPop(&Pdo->PreparedReqs, &Request->Entry);
- return FALSE; // ring full
- }
-
- return TRUE;
-}
-
-static FORCEINLINE VOID
-PdoCompleteShutdown(
- __in PXENVBD_PDO Pdo
- )
-{
- if (QueueCount(&Pdo->ShutdownSrbs) == 0)
- return;
-
- if (QueueCount(&Pdo->FreshSrbs) ||
- QueueCount(&Pdo->PreparedReqs) ||
- QueueCount(&Pdo->SubmittedReqs))
- return;
-
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PLIST_ENTRY Entry = QueuePop(&Pdo->ShutdownSrbs);
- if (Entry == NULL)
- break;
- SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
- SrbExt->Srb->SrbStatus = SRB_STATUS_SUCCESS;
- AdapterCompleteSrb(PdoGetAdapter(Pdo), SrbExt->Srb);
- }
-}
-
-static FORCEINLINE PCHAR
-BlkifOperationName(
- IN UCHAR Operation
- )
-{
- switch (Operation) {
- case BLKIF_OP_READ: return "READ";
- case BLKIF_OP_WRITE: return "WRITE";
- case BLKIF_OP_WRITE_BARRIER: return "WRITE_BARRIER";
- case BLKIF_OP_FLUSH_DISKCACHE: return "FLUSH_DISKCACHE";
- case BLKIF_OP_RESERVED_1: return "RESERVED_1";
- case BLKIF_OP_DISCARD: return "DISCARD";
- case BLKIF_OP_INDIRECT: return "INDIRECT";
- default: return "<unknown>";
- }
-}
-
-VOID
-PdoSubmitRequests(
- __in PXENVBD_PDO Pdo
- )
-{
- for (;;) {
- // submit all prepared requests (0 or more requests)
- // return TRUE if submitted 0 or more requests from prepared queue
- // return FALSE iff ring is full
- if (!PdoSubmitPrepared(Pdo))
- break;
-
- // prepare a single SRB (into 1 or more requests)
- // return TRUE if prepare succeeded
- // return FALSE if prepare failed or fresh queue empty
- if (!PdoPrepareFresh(Pdo))
- break;
- }
-
- // if no requests/SRBs outstanding, complete any shutdown SRBs
- PdoCompleteShutdown(Pdo);
-}
-
-VOID
-PdoCompleteResponse(
- __in PXENVBD_PDO Pdo,
- __in ULONG Tag,
- __in SHORT Status
- )
-{
- PXENVBD_REQUEST Request;
- PSCSI_REQUEST_BLOCK Srb;
- PXENVBD_SRBEXT SrbExt;
-
- Request = PdoRequestFromTag(Pdo, Tag);
- if (Request == NULL)
- return;
-
- Srb = Request->Srb;
- SrbExt = GetSrbExt(Srb);
- ASSERT3P(SrbExt, !=, NULL);
-
- switch (Status) {
- case BLKIF_RSP_OKAY:
- RequestCopyOutput(Request);
- break;
-
- case BLKIF_RSP_EOPNOTSUPP:
- // Remove appropriate feature support
- FrontendRemoveFeature(Pdo->Frontend, Request->Operation);
- // Succeed this SRB, subsiquent SRBs will be succeeded instead of being passed to the backend.
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- break;
-
- case BLKIF_RSP_ERROR:
- default:
- Warning("Target[%d] : %s BLKIF_RSP_ERROR (Tag %x)\n",
- PdoGetTargetId(Pdo), BlkifOperationName(Request->Operation), Tag);
- Srb->SrbStatus = SRB_STATUS_ERROR;
- break;
- }
-
- PdoPutRequest(Pdo, Request);
-
- // complete srb
- if (InterlockedDecrement(&SrbExt->Count) == 0) {
- if (Srb->SrbStatus == SRB_STATUS_PENDING) {
- // SRB has not hit a failure condition (BLKIF_RSP_ERROR | BLKIF_RSP_EOPNOTSUPP)
- // from any of its responses. SRB must have succeeded
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- Srb->ScsiStatus = 0x00; // SCSI_GOOD
- } else {
- // Srb->SrbStatus has already been set by 1 or more requests with Status != BLKIF_RSP_OKAY
- Srb->ScsiStatus = 0x40; // SCSI_ABORTED
- }
-
- AdapterCompleteSrb(PdoGetAdapter(Pdo), Srb);
- }
-}
-
-VOID
-PdoPreResume(
- __in PXENVBD_PDO Pdo
- )
-{
- LIST_ENTRY List;
-
- InitializeListHead(&List);
-
- // pop all submitted requests, cleanup and add associated SRB to a list
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PXENVBD_REQUEST Request;
- PLIST_ENTRY Entry = QueuePop(&Pdo->SubmittedReqs);
- if (Entry == NULL)
- break;
- Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
- SrbExt = GetSrbExt(Request->Srb);
-
- PdoPutRequest(Pdo, Request);
-
- if (InterlockedDecrement(&SrbExt->Count) == 0) {
- InsertTailList(&List, &SrbExt->Entry);
- }
- }
-
- // pop all prepared requests, cleanup and add associated SRB to a list
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PXENVBD_REQUEST Request;
- PLIST_ENTRY Entry = QueuePop(&Pdo->PreparedReqs);
- if (Entry == NULL)
- break;
- Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
- SrbExt = GetSrbExt(Request->Srb);
-
- PdoPutRequest(Pdo, Request);
-
- if (InterlockedDecrement(&SrbExt->Count) == 0) {
- InsertTailList(&List, &SrbExt->Entry);
- }
- }
-
- // foreach SRB in list, put on start of FreshSrbs
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PLIST_ENTRY Entry = RemoveTailList(&List);
- if (Entry == &List)
- break;
- SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
-
- QueueUnPop(&Pdo->FreshSrbs, &SrbExt->Entry);
- }
-
- // now the first set of requests popped off submitted list is the next SRB
- // to be popped off the fresh list
-}
-
-VOID
-PdoPostResume(
- __in PXENVBD_PDO Pdo
- )
-{
- KIRQL Irql;
-
- Verbose("Target[%d] : %d Fresh SRBs\n", PdoGetTargetId(Pdo), QueueCount(&Pdo->FreshSrbs));
-
- // clear missing flag
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- Verbose("Target[%d] : %s (%s)\n", PdoGetTargetId(Pdo), Pdo->Missing ? "MISSING" : "NOT_MISSING", Pdo->Reason);
- Pdo->Missing = FALSE;
- Pdo->Reason = NULL;
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-}
-
-//=============================================================================
-// SRBs
-__checkReturn
-static FORCEINLINE BOOLEAN
-__ValidateSectors(
- __in ULONG64 SectorCount,
- __in ULONG64 Start,
- __in ULONG Length
- )
-{
- // Deal with overflow
- return (Start < SectorCount) && ((Start + Length) <= SectorCount);
-}
-
-__checkReturn
-static FORCEINLINE BOOLEAN
-__ValidateSrbBuffer(
- __in PCHAR Caller,
- __in PSCSI_REQUEST_BLOCK Srb,
- __in ULONG MinLength
- )
-{
- if (Srb->DataBuffer == NULL) {
- Error("%s: Srb[0x%p].DataBuffer = NULL\n", Caller, Srb);
- return FALSE;
- }
- if (MinLength) {
- if (Srb->DataTransferLength < MinLength) {
- Error("%s: Srb[0x%p].DataTransferLength < %d\n", Caller, Srb, MinLength);
- return FALSE;
- }
- } else {
- if (Srb->DataTransferLength == 0) {
- Error("%s: Srb[0x%p].DataTransferLength = 0\n", Caller, Srb);
- return FALSE;
- }
- }
-
- return TRUE;
-}
-
-__checkReturn
-static DECLSPEC_NOINLINE BOOLEAN
-PdoReadWrite(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PXENVBD_DISKINFO DiskInfo = FrontendGetDiskInfo(Pdo->Frontend);
- PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
- PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Pdo->Frontend);
-
- if (FrontendGetCaps(Pdo->Frontend)->Connected == FALSE) {
- Trace("Target[%d] : Not Ready, fail SRB\n", PdoGetTargetId(Pdo));
- Srb->ScsiStatus = 0x40; // SCSI_ABORT;
- return TRUE;
- }
-
- // check valid sectors
- if (!__ValidateSectors(DiskInfo->SectorCount, Cdb_LogicalBlock(Srb), Cdb_TransferBlock(Srb))) {
- Trace("Target[%d] : Invalid Sector (%d @ %lld < %lld)\n", PdoGetTargetId(Pdo), Cdb_TransferBlock(Srb), Cdb_LogicalBlock(Srb), DiskInfo->SectorCount);
- Srb->ScsiStatus = 0x40; // SCSI_ABORT
- return TRUE; // Complete now
- }
-
- QueueAppend(&Pdo->FreshSrbs, &SrbExt->Entry);
- NotifierKick(Notifier);
-
- return FALSE;
-}
-
-__checkReturn
-static DECLSPEC_NOINLINE BOOLEAN
-PdoSyncCache(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
- PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Pdo->Frontend);
-
- if (FrontendGetCaps(Pdo->Frontend)->Connected == FALSE) {
- Trace("Target[%d] : Not Ready, fail SRB\n", PdoGetTargetId(Pdo));
- Srb->ScsiStatus = 0x40; // SCSI_ABORT;
- return TRUE;
- }
-
- if (FrontendGetDiskInfo(Pdo->Frontend)->FlushCache == FALSE &&
- FrontendGetDiskInfo(Pdo->Frontend)->Barrier == FALSE) {
- Trace("Target[%d] : FLUSH and BARRIER not supported, suppressing\n", PdoGetTargetId(Pdo));
- Srb->ScsiStatus = 0x00; // SCSI_GOOD
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- return TRUE;
- }
-
- QueueAppend(&Pdo->FreshSrbs, &SrbExt->Entry);
- NotifierKick(Notifier);
-
- return FALSE;
-}
-
-__checkReturn
-static DECLSPEC_NOINLINE BOOLEAN
-PdoUnmap(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
- PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Pdo->Frontend);
-
- if (FrontendGetCaps(Pdo->Frontend)->Connected == FALSE) {
- Trace("Target[%d] : Not Ready, fail SRB\n", PdoGetTargetId(Pdo));
- Srb->ScsiStatus = 0x40; // SCSI_ABORT;
- return TRUE;
- }
-
- if (FrontendGetDiskInfo(Pdo->Frontend)->Discard == FALSE) {
- Trace("Target[%d] : DISCARD not supported, suppressing\n", PdoGetTargetId(Pdo));
- Srb->ScsiStatus = 0x00; // SCSI_GOOD
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- return TRUE;
- }
-
- QueueAppend(&Pdo->FreshSrbs, &SrbExt->Entry);
- NotifierKick(Notifier);
-
- return FALSE;
-}
-
-#define MODE_CACHING_PAGE_LENGTH 20
-static DECLSPEC_NOINLINE VOID
-PdoModeSense(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PMODE_PARAMETER_HEADER Header = Srb->DataBuffer;
- const UCHAR PageCode = Cdb_PageCode(Srb);
- ULONG LengthLeft = Cdb_AllocationLength(Srb);
- PVOID CurrentPage = Srb->DataBuffer;
-
- UNREFERENCED_PARAMETER(Pdo);
-
- RtlZeroMemory(Srb->DataBuffer, Srb->DataTransferLength);
-
- if (!__ValidateSrbBuffer(__FUNCTION__, Srb, (ULONG)sizeof(struct _MODE_SENSE))) {
- Srb->ScsiStatus = 0x40;
- Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
- Srb->DataTransferLength = 0;
- return;
- }
-
- // TODO : CDROM requires more ModePage entries
- // Header
- Header->ModeDataLength = sizeof(MODE_PARAMETER_HEADER) - 1;
- Header->MediumType = 0;
- Header->DeviceSpecificParameter = 0;
- Header->BlockDescriptorLength = 0;
- LengthLeft -= sizeof(MODE_PARAMETER_HEADER);
- CurrentPage = ((PUCHAR)CurrentPage + sizeof(MODE_PARAMETER_HEADER));
-
- // Fill in Block Parameters (if Specified and space)
- // when the DBD (Disable Block Descriptor) is set, ignore the block page
- if (Cdb_Dbd(Srb) == 0 &&
- LengthLeft >= sizeof(MODE_PARAMETER_BLOCK)) {
- PMODE_PARAMETER_BLOCK Block = (PMODE_PARAMETER_BLOCK)CurrentPage;
- // Fill in BlockParams
- Block->DensityCode = 0;
- Block->NumberOfBlocks[0] = 0;
- Block->NumberOfBlocks[1] = 0;
- Block->NumberOfBlocks[2] = 0;
- Block->BlockLength[0] = 0;
- Block->BlockLength[1] = 0;
- Block->BlockLength[2] = 0;
-
- Header->BlockDescriptorLength = sizeof(MODE_PARAMETER_BLOCK);
- Header->ModeDataLength += sizeof(MODE_PARAMETER_BLOCK);
- LengthLeft -= sizeof(MODE_PARAMETER_BLOCK);
- CurrentPage = ((PUCHAR)CurrentPage + sizeof(MODE_PARAMETER_BLOCK));
- }
-
- // Fill in Cache Parameters (if Specified and space)
- if ((PageCode == MODE_PAGE_CACHING || PageCode == MODE_SENSE_RETURN_ALL) &&
- LengthLeft >= MODE_CACHING_PAGE_LENGTH) {
- PMODE_CACHING_PAGE Caching = (PMODE_CACHING_PAGE)CurrentPage;
- // Fill in CachingParams
- Caching->PageCode = MODE_PAGE_CACHING;
- Caching->PageSavable = 0;
- Caching->PageLength = MODE_CACHING_PAGE_LENGTH;
- Caching->ReadDisableCache = 0;
- Caching->MultiplicationFactor = 0;
- Caching->WriteCacheEnable = FrontendGetDiskInfo(Pdo->Frontend)->FlushCache ? 1 : 0;
- Caching->WriteRetensionPriority = 0;
- Caching->ReadRetensionPriority = 0;
- Caching->DisablePrefetchTransfer[0] = 0;
- Caching->DisablePrefetchTransfer[1] = 0;
- Caching->MinimumPrefetch[0] = 0;
- Caching->MinimumPrefetch[1] = 0;
- Caching->MaximumPrefetch[0] = 0;
- Caching->MaximumPrefetch[1] = 0;
- Caching->MaximumPrefetchCeiling[0] = 0;
- Caching->MaximumPrefetchCeiling[1] = 0;
-
- Header->ModeDataLength += MODE_CACHING_PAGE_LENGTH;
- LengthLeft -= MODE_CACHING_PAGE_LENGTH;
- CurrentPage = ((PUCHAR)CurrentPage + MODE_CACHING_PAGE_LENGTH);
- }
-
- // Fill in Informational Exception Parameters (if Specified and space)
- if ((PageCode == MODE_PAGE_FAULT_REPORTING || PageCode == MODE_SENSE_RETURN_ALL) &&
- LengthLeft >= sizeof(MODE_INFO_EXCEPTIONS)) {
- PMODE_INFO_EXCEPTIONS Exceptions = (PMODE_INFO_EXCEPTIONS)CurrentPage;
- // Fill in Exceptions
- Exceptions->PageCode = MODE_PAGE_FAULT_REPORTING;
- Exceptions->PSBit = 0;
- Exceptions->PageLength = sizeof(MODE_INFO_EXCEPTIONS);
- Exceptions->Flags = 0;
- Exceptions->Dexcpt = 1; // disabled
- Exceptions->ReportMethod = 0;
- Exceptions->IntervalTimer[0] = 0;
- Exceptions->IntervalTimer[1] = 0;
- Exceptions->IntervalTimer[2] = 0;
- Exceptions->IntervalTimer[3] = 0;
- Exceptions->ReportCount[0] = 0;
- Exceptions->ReportCount[1] = 0;
- Exceptions->ReportCount[2] = 0;
- Exceptions->ReportCount[3] = 0;
-
- Header->ModeDataLength += sizeof(MODE_INFO_EXCEPTIONS);
- LengthLeft -= sizeof(MODE_INFO_EXCEPTIONS);
- CurrentPage = ((PUCHAR)CurrentPage + sizeof(MODE_INFO_EXCEPTIONS));
- }
-
- // Finish this SRB
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- Srb->DataTransferLength = __min(Cdb_AllocationLength(Srb), (ULONG)(Header->ModeDataLength + 1));
-}
-
-static DECLSPEC_NOINLINE VOID
-PdoRequestSense(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PSENSE_DATA Sense = Srb->DataBuffer;
-
- UNREFERENCED_PARAMETER(Pdo);
-
- if (!__ValidateSrbBuffer(__FUNCTION__, Srb, (ULONG)sizeof(SENSE_DATA))) {
- Srb->ScsiStatus = 0x40;
- Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
- return;
- }
-
- RtlZeroMemory(Sense, sizeof(SENSE_DATA));
-
- Sense->ErrorCode = 0x70;
- Sense->Valid = 1;
- Sense->AdditionalSenseCodeQualifier = 0;
- Sense->SenseKey = SCSI_SENSE_NO_SENSE;
- Sense->AdditionalSenseCode = SCSI_ADSENSE_NO_SENSE;
- Srb->DataTransferLength = sizeof(SENSE_DATA);
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
-}
-
-static DECLSPEC_NOINLINE VOID
-PdoReportLuns(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- ULONG Length;
- ULONG Offset;
- ULONG AllocLength = Cdb_AllocationLength(Srb);
- PUCHAR Buffer = Srb->DataBuffer;
-
- UNREFERENCED_PARAMETER(Pdo);
-
- if (!__ValidateSrbBuffer(__FUNCTION__, Srb, 8)) {
- Srb->ScsiStatus = 0x40;
- Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
- Srb->DataTransferLength = 0;
- return;
- }
-
- RtlZeroMemory(Buffer, AllocLength);
-
- Length = 0;
- Offset = 8;
-
- if (Offset + 8 <= AllocLength) {
- Buffer[Offset] = 0;
- Offset += 8;
- Length += 8;
- }
-
- if (Offset + 8 <= AllocLength) {
- Buffer[Offset] = XENVBD_MAX_TARGETS;
- Offset += 8;
- Length += 8;
- }
-
- REVERSE_BYTES(Buffer, &Length);
-
- Srb->DataTransferLength = __min(Length, AllocLength);
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
-}
-
-static DECLSPEC_NOINLINE VOID
-PdoReadCapacity(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PREAD_CAPACITY_DATA Capacity = Srb->DataBuffer;
- PXENVBD_DISKINFO DiskInfo = FrontendGetDiskInfo(Pdo->Frontend);
- ULONG64 SectorCount;
- ULONG SectorSize;
- ULONG LastBlock;
-
- if (Cdb_PMI(Srb) == 0 && Cdb_LogicalBlock(Srb) != 0) {
- Srb->ScsiStatus = 0x02; // CHECK_CONDITION
- return;
- }
-
- SectorCount = DiskInfo->SectorCount;
- SectorSize = DiskInfo->SectorSize;
-
- if (SectorCount == (ULONG)SectorCount)
- LastBlock = (ULONG)SectorCount - 1;
- else
- LastBlock = ~(ULONG)0;
-
- if (Capacity) {
- Capacity->LogicalBlockAddress = _byteswap_ulong(LastBlock);
- Capacity->BytesPerBlock = _byteswap_ulong(SectorSize);
- }
-
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
-}
-
-static DECLSPEC_NOINLINE VOID
-PdoReadCapacity16(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PREAD_CAPACITY16_DATA Capacity = Srb->DataBuffer;
- PXENVBD_DISKINFO DiskInfo = FrontendGetDiskInfo(Pdo->Frontend);
- ULONG64 SectorCount;
- ULONG SectorSize;
- ULONG PhysSectorSize;
- ULONG LogicalPerPhysical;
- ULONG LogicalPerPhysicalExponent;
-
- if (Cdb_PMI(Srb) == 0 && Cdb_LogicalBlock(Srb) != 0) {
- Srb->ScsiStatus = 0x02; // CHECK_CONDITION
- return;
- }
-
- SectorCount = DiskInfo->SectorCount;
- SectorSize = DiskInfo->SectorSize;
- PhysSectorSize = DiskInfo->PhysSectorSize;
-
- LogicalPerPhysical = PhysSectorSize / SectorSize;
-
- if (!_BitScanReverse(&LogicalPerPhysicalExponent, LogicalPerPhysical))
- LogicalPerPhysicalExponent = 0;
-
- if (Capacity) {
- Capacity->LogicalBlockAddress.QuadPart = _byteswap_uint64(SectorCount - 1);
- Capacity->BytesPerBlock = _byteswap_ulong(SectorSize);
- Capacity->LogicalPerPhysicalExponent = (UCHAR)LogicalPerPhysicalExponent;
- }
-
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
-}
-
-//=============================================================================
-// StorPort Methods
-__checkReturn
-static FORCEINLINE BOOLEAN
-__PdoExecuteScsi(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- const UCHAR Operation = Cdb_OperationEx(Srb);
- PXENVBD_DISKINFO DiskInfo = FrontendGetDiskInfo(Pdo->Frontend);
-
- if (DiskInfo->DiskInfo & VDISK_READONLY) {
- Trace("Target[%d] : (%08x) Read-Only, fail SRB (%02x:%s)\n", PdoGetTargetId(Pdo),
- DiskInfo->DiskInfo, Operation, Cdb_OperationName(Operation));
- Srb->ScsiStatus = 0x40; // SCSI_ABORT
- return TRUE;
- }
-
- // idea: check pdo state here. still push to freshsrbs
- switch (Operation) {
- case SCSIOP_READ:
- case SCSIOP_WRITE:
- return PdoReadWrite(Pdo, Srb);
- break;
-
- case SCSIOP_SYNCHRONIZE_CACHE:
- return PdoSyncCache(Pdo, Srb);
- break;
-
- case SCSIOP_UNMAP:
- return PdoUnmap(Pdo, Srb);
- break;
-
- case SCSIOP_INQUIRY:
- if (!StorPortSetDeviceQueueDepth(PdoGetAdapter(Pdo),
- 0,
- (UCHAR)PdoGetTargetId(Pdo),
- 0,
- XENVBD_MAX_QUEUE_DEPTH))
- Verbose("Target[%d] : Failed to set queue depth\n",
- PdoGetTargetId(Pdo));
- PdoInquiry(PdoGetTargetId(Pdo), FrontendGetInquiry(Pdo->Frontend), Srb, Pdo->DeviceType);
- break;
- case SCSIOP_MODE_SENSE:
- PdoModeSense(Pdo, Srb);
- break;
- case SCSIOP_REQUEST_SENSE:
- PdoRequestSense(Pdo, Srb);
- break;
- case SCSIOP_REPORT_LUNS:
- PdoReportLuns(Pdo, Srb);
- break;
- case SCSIOP_READ_CAPACITY:
- PdoReadCapacity(Pdo, Srb);
- break;
- case SCSIOP_READ_CAPACITY16:
- PdoReadCapacity16(Pdo, Srb);
- break;
- case SCSIOP_MEDIUM_REMOVAL:
- case SCSIOP_TEST_UNIT_READY:
- case SCSIOP_RESERVE_UNIT:
- case SCSIOP_RESERVE_UNIT10:
- case SCSIOP_RELEASE_UNIT:
- case SCSIOP_RELEASE_UNIT10:
- case SCSIOP_VERIFY:
- case SCSIOP_VERIFY16:
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- break;
- case SCSIOP_START_STOP_UNIT:
- Trace("Target[%d] : Start/Stop Unit (%02X)\n", PdoGetTargetId(Pdo), Srb->Cdb[4]);
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
- break;
- default:
- Trace("Target[%d] : Unsupported CDB (%02x:%s)\n", PdoGetTargetId(Pdo), Operation, Cdb_OperationName(Operation));
- break;
- }
- return TRUE;
-}
-
-static FORCEINLINE BOOLEAN
-__PdoQueueShutdown(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
- PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Pdo->Frontend);
-
- QueueAppend(&Pdo->ShutdownSrbs, &SrbExt->Entry);
- NotifierKick(Notifier);
-
- return FALSE;
-}
-
-static FORCEINLINE BOOLEAN
-__PdoReset(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- Verbose("Target[%u] ====>\n", PdoGetTargetId(Pdo));
-
- PdoReset(Pdo);
- Srb->SrbStatus = SRB_STATUS_SUCCESS;
-
- Verbose("Target[%u] <====\n", PdoGetTargetId(Pdo));
- return TRUE;
-}
-
-__checkReturn
-static FORCEINLINE BOOLEAN
-__ValidateSrbForPdo(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- const UCHAR Operation = Cdb_OperationEx(Srb);
-
- if (Pdo == NULL) {
- Error("Invalid Pdo(NULL) (%02x:%s)\n",
- Operation, Cdb_OperationName(Operation));
- Srb->SrbStatus = SRB_STATUS_INVALID_TARGET_ID;
- return FALSE;
- }
-
- if (Srb->PathId != 0) {
- Error("Target[%d] : Invalid PathId(%d) (%02x:%s)\n",
- PdoGetTargetId(Pdo), Srb->PathId, Operation, Cdb_OperationName(Operation));
- Srb->SrbStatus = SRB_STATUS_INVALID_PATH_ID;
- return FALSE;
- }
-
- if (Srb->Lun != 0) {
- Error("Target[%d] : Invalid Lun(%d) (%02x:%s)\n",
- PdoGetTargetId(Pdo), Srb->Lun, Operation, Cdb_OperationName(Operation));
- Srb->SrbStatus = SRB_STATUS_INVALID_LUN;
- return FALSE;
- }
-
- if (PdoIsMissing(Pdo)) {
- Error("Target[%d] : %s (%s) (%02x:%s)\n",
- PdoGetTargetId(Pdo), Pdo->Missing ? "MISSING" : "NOT_MISSING", Pdo->Reason, Operation, Cdb_OperationName(Operation));
- Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
- return FALSE;
- }
-
- return TRUE;
-}
-
-__checkReturn
-BOOLEAN
-PdoStartIo(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- )
-{
- if (!__ValidateSrbForPdo(Pdo, Srb))
- return TRUE;
-
- switch (Srb->Function) {
- case SRB_FUNCTION_EXECUTE_SCSI:
- return __PdoExecuteScsi(Pdo, Srb);
-
- case SRB_FUNCTION_RESET_DEVICE:
- return __PdoReset(Pdo, Srb);
-
- case SRB_FUNCTION_FLUSH:
- case SRB_FUNCTION_SHUTDOWN:
- return __PdoQueueShutdown(Pdo, Srb);
-
- default:
- return TRUE;
- }
-}
-
-static FORCEINLINE VOID
-__PdoCleanupSubmittedReqs(
- IN PXENVBD_PDO Pdo
- )
-{
- // Fail PreparedReqs
- for (;;) {
- PXENVBD_SRBEXT SrbExt;
- PXENVBD_REQUEST Request;
- PLIST_ENTRY Entry = QueuePop(&Pdo->SubmittedReqs);
- if (Entry == NULL)
- break;
- Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
- SrbExt = GetSrbExt(Request->Srb);
-
- Verbose("Target[%d] : SubmittedReq 0x%p -> FAILED\n", PdoGetTargetId(Pdo), Request);
-
- PdoPutRequest(Pdo, Request);
-
- if (InterlockedDecrement(&SrbExt->Count) == 0) {
- SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
- SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED
- AdapterCompleteSrb(PdoGetAdapter(Pdo), SrbExt->Srb);
- }
- }
-}
-
-VOID
-PdoReset(
- __in PXENVBD_PDO Pdo
- )
-{
- Trace("Target[%d] ====> (Irql=%d)\n", PdoGetTargetId(Pdo), KeGetCurrentIrql());
-
- __PdoPauseDataPath(Pdo, TRUE);
-
- if (QueueCount(&Pdo->SubmittedReqs)) {
- Error("Target[%d] : backend has %u outstanding requests after a PdoReset\n",
- PdoGetTargetId(Pdo), QueueCount(&Pdo->SubmittedReqs));
- }
-
- __PdoUnpauseDataPath(Pdo);
-
- Trace("Target[%d] <==== (Irql=%d)\n", PdoGetTargetId(Pdo), KeGetCurrentIrql());
-}
-
-
-VOID
-PdoSrbPnp(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_PNP_REQUEST_BLOCK Srb
- )
-{
- switch (Srb->PnPAction) {
- case StorQueryCapabilities: {
- PSTOR_DEVICE_CAPABILITIES DeviceCaps = Srb->DataBuffer;
- PXENVBD_CAPS Caps = FrontendGetCaps(Pdo->Frontend);
-
- if (Caps->Removable)
- DeviceCaps->Removable = 1;
- if (Caps->Removable)
- DeviceCaps->EjectSupported = 1;
- if (Caps->SurpriseRemovable)
- DeviceCaps->SurpriseRemovalOK = 1;
-
- DeviceCaps->UniqueID = 1;
-
- } break;
-
- default:
- break;
- }
-}
-
-//=============================================================================
-// PnP Handler
-static FORCEINLINE VOID
-__PdoDeviceUsageNotification(
- __in PXENVBD_PDO Pdo,
- __in PIRP Irp
- )
-{
- PIO_STACK_LOCATION StackLocation;
- BOOLEAN Value;
- DEVICE_USAGE_NOTIFICATION_TYPE Type;
- PXENVBD_CAPS Caps = FrontendGetCaps(Pdo->Frontend);
-
- StackLocation = IoGetCurrentIrpStackLocation(Irp);
- Value = StackLocation->Parameters.UsageNotification.InPath;
- Type = StackLocation->Parameters.UsageNotification.Type;
-
- switch (Type) {
- case DeviceUsageTypePaging:
- if (Caps->Paging == Value)
- return;
- Caps->Paging = Value;
- break;
-
- case DeviceUsageTypeHibernation:
- if (Caps->Hibernation == Value)
- return;
- Caps->Hibernation = Value;
- break;
-
- case DeviceUsageTypeDumpFile:
- if (Caps->DumpFile == Value)
- return;
- Caps->DumpFile = Value;
- break;
-
- default:
- return;
- }
- FrontendWriteUsage(Pdo->Frontend);
-}
-
-static FORCEINLINE VOID
-__PdoCheckEjectPending(
- __in PXENVBD_PDO Pdo
- )
-{
- KIRQL Irql;
- BOOLEAN EjectPending = FALSE;
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- if (Pdo->EjectPending) {
- EjectPending = TRUE;
- Pdo->EjectPending = FALSE;
- Pdo->EjectRequested = TRUE;
- }
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-
- if (EjectPending) {
- Verbose("Target[%d] : IoRequestDeviceEject(0x%p)\n", PdoGetTargetId(Pdo), Pdo->DeviceObject);
- IoRequestDeviceEject(Pdo->DeviceObject);
- }
-}
-
-static FORCEINLINE VOID
-__PdoCheckEjectFailed(
- __in PXENVBD_PDO Pdo
- )
-{
- KIRQL Irql;
- BOOLEAN EjectFailed = FALSE;
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- if (Pdo->EjectRequested) {
- EjectFailed = TRUE;
- Pdo->EjectRequested = FALSE;
- }
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-
- if (EjectFailed) {
- Error("Target[%d] : Unplug failed due to open handle(s)!\n", PdoGetTargetId(Pdo));
- FrontendStoreWriteFrontend(Pdo->Frontend, "error", "Unplug failed due to open handle(s)!");
- }
-}
-
-static FORCEINLINE VOID
-__PdoRemoveDevice(
- __in PXENVBD_PDO Pdo
- )
-{
- PdoD0ToD3(Pdo);
-
- switch (PdoGetDevicePnpState(Pdo)) {
- case SurpriseRemovePending:
- PdoSetMissing(Pdo, "Surprise Remove");
- PdoSetDevicePnpState(Pdo, Deleted);
- StorPortNotification(BusChangeDetected, PdoGetAdapter(Pdo), 0);
- break;
-
- default:
- PdoSetMissing(Pdo, "Removed");
- PdoSetDevicePnpState(Pdo, Deleted);
- StorPortNotification(BusChangeDetected, PdoGetAdapter(Pdo), 0);
- break;
- }
-}
-
-static FORCEINLINE VOID
-__PdoEject(
- __in PXENVBD_PDO Pdo
- )
-{
- PdoSetMissing(Pdo, "Ejected");
- PdoSetDevicePnpState(Pdo, Deleted);
- StorPortNotification(BusChangeDetected, PdoGetAdapter(Pdo), 0);
-}
-
-__checkReturn
-NTSTATUS
-PdoDispatchPnp(
- __in PXENVBD_PDO Pdo,
- __in PDEVICE_OBJECT DeviceObject,
- __in PIRP Irp
- )
-{
- PIO_STACK_LOCATION Stack = IoGetCurrentIrpStackLocation(Irp);
-
- __PdoCheckEjectPending(Pdo);
-
- switch (Stack->MinorFunction) {
- case IRP_MN_START_DEVICE:
- (VOID) PdoD3ToD0(Pdo);
- PdoSetDevicePnpState(Pdo, Started);
- break;
-
- case IRP_MN_QUERY_STOP_DEVICE:
- PdoSetDevicePnpState(Pdo, StopPending);
- break;
-
- case IRP_MN_CANCEL_STOP_DEVICE:
- __PdoRestoreDevicePnpState(Pdo, StopPending);
- break;
-
- case IRP_MN_STOP_DEVICE:
- PdoD0ToD3(Pdo);
- PdoSetDevicePnpState(Pdo, Stopped);
- break;
-
- case IRP_MN_QUERY_REMOVE_DEVICE:
- PdoSetDevicePnpState(Pdo, RemovePending);
- break;
-
- case IRP_MN_CANCEL_REMOVE_DEVICE:
- __PdoCheckEjectFailed(Pdo);
- __PdoRestoreDevicePnpState(Pdo, RemovePending);
- break;
-
- case IRP_MN_SURPRISE_REMOVAL:
- PdoSetDevicePnpState(Pdo, SurpriseRemovePending);
- break;
-
- case IRP_MN_REMOVE_DEVICE:
- __PdoRemoveDevice(Pdo);
- break;
-
- case IRP_MN_EJECT:
- __PdoEject(Pdo);
- break;
-
- case IRP_MN_DEVICE_USAGE_NOTIFICATION:
- __PdoDeviceUsageNotification(Pdo, Irp);
- break;
-
- default:
- break;
- }
- PdoDereference(Pdo);
- return DriverDispatchPnp(DeviceObject, Irp);
-}
-
-__drv_maxIRQL(DISPATCH_LEVEL)
-VOID
-PdoIssueDeviceEject(
- __in PXENVBD_PDO Pdo,
- __in __nullterminated const CHAR* Reason
- )
-{
- KIRQL Irql;
- BOOLEAN DoEject = FALSE;
-
- KeAcquireSpinLock(&Pdo->Lock, &Irql);
- if (Pdo->DeviceObject) {
- DoEject = TRUE;
- Pdo->EjectRequested = TRUE;
- } else {
- Pdo->EjectPending = TRUE;
- }
- KeReleaseSpinLock(&Pdo->Lock, Irql);
-
- Verbose("Target[%d] : Ejecting (%s - %s)\n", PdoGetTargetId(Pdo), DoEject ? "Now" : "Next PnP IRP", Reason);
- if (!Pdo->WrittenEjected) {
- Pdo->WrittenEjected = TRUE;
- FrontendStoreWriteFrontend(Pdo->Frontend, "ejected", "1");
- }
- if (DoEject) {
- Verbose("Target[%d] : IoRequestDeviceEject(0x%p)\n", PdoGetTargetId(Pdo), Pdo->DeviceObject);
- IoRequestDeviceEject(Pdo->DeviceObject);
- } else {
- Verbose("Target[%d] : Triggering BusChangeDetected to detect device\n", PdoGetTargetId(Pdo));
- StorPortNotification(BusChangeDetected, PdoGetAdapter(Pdo), 0);
- }
-}
-
-__checkReturn
-NTSTATUS
-PdoD3ToD0(
- __in PXENVBD_PDO Pdo
- )
-{
- NTSTATUS Status;
- const ULONG TargetId = PdoGetTargetId(Pdo);
-
- if (!PdoSetDevicePowerState(Pdo, PowerDeviceD0))
- return STATUS_SUCCESS;
-
- Trace("Target[%d] @ (%d) =====>\n", TargetId, KeGetCurrentIrql());
- Verbose("Target[%d] : D3->D0\n", TargetId);
-
- // power up frontend
- Status = FrontendD3ToD0(Pdo->Frontend);
- if (!NT_SUCCESS(Status))
- goto fail1;
-
- // connect frontend
- Status = FrontendSetState(Pdo->Frontend, XENVBD_ENABLED);
- if (!NT_SUCCESS(Status))
- goto fail2;
- __PdoUnpauseDataPath(Pdo);
-
- Trace("Target[%d] @ (%d) <=====\n", TargetId, KeGetCurrentIrql());
- return STATUS_SUCCESS;
-
-fail2:
- Error("Fail2\n");
- FrontendD0ToD3(Pdo->Frontend);
-
-fail1:
- Error("Fail1 (%08x)\n", Status);
-
- Pdo->DevicePowerState = PowerDeviceD3;
-
- return Status;
-}
-
-VOID
-PdoD0ToD3(
- __in PXENVBD_PDO Pdo
- )
-{
- const ULONG TargetId = PdoGetTargetId(Pdo);
-
- if (!PdoSetDevicePowerState(Pdo, PowerDeviceD3))
- return;
-
- Trace("Target[%d] @ (%d) =====>\n", TargetId, KeGetCurrentIrql());
- Verbose("Target[%d] : D0->D3\n", TargetId);
-
- // close frontend
- __PdoPauseDataPath(Pdo, FALSE);
- (VOID) FrontendSetState(Pdo->Frontend, XENVBD_CLOSED);
- ASSERT3U(QueueCount(&Pdo->SubmittedReqs), ==, 0);
-
- // power down frontend
- FrontendD0ToD3(Pdo->Frontend);
-
- Trace("Target[%d] @ (%d) <=====\n", TargetId, KeGetCurrentIrql());
-}
-
-__checkReturn
-BOOLEAN
-PdoCreate(
- __in PXENVBD_ADAPTER Adapter,
- __in __nullterminated PCHAR DeviceId,
- __in ULONG TargetId,
- __in XENVBD_DEVICE_TYPE DeviceType
- )
-{
- NTSTATUS Status;
- PXENVBD_PDO Pdo;
-
- Trace("Target[%d] @ (%d) =====>\n", TargetId, KeGetCurrentIrql());
-
- Status = STATUS_INSUFFICIENT_RESOURCES;
-#pragma warning(suppress: 6014)
- Pdo = __PdoAlloc(sizeof(XENVBD_PDO));
- if (!Pdo)
- goto fail1;
-
- Verbose("Target[%d] : Creating\n", TargetId);
- Pdo->Signature = PDO_SIGNATURE;
- Pdo->Adapter = Adapter;
- Pdo->DeviceObject = NULL; // filled in later
- KeInitializeEvent(&Pdo->RemoveEvent, SynchronizationEvent, FALSE);
- Pdo->ReferenceCount = 1;
- Pdo->Paused = 1; // Paused until D3->D0 transition
- Pdo->DevicePnpState = Present;
- Pdo->DevicePowerState = PowerDeviceD3;
- Pdo->DeviceType = DeviceType;
-
- KeInitializeSpinLock(&Pdo->Lock);
- QueueInit(&Pdo->FreshSrbs);
- QueueInit(&Pdo->PreparedReqs);
- QueueInit(&Pdo->SubmittedReqs);
- QueueInit(&Pdo->ShutdownSrbs);
- __LookasideInit(&Pdo->RequestList, sizeof(XENVBD_REQUEST), REQUEST_POOL_TAG);
- __LookasideInit(&Pdo->SegmentList, sizeof(XENVBD_SEGMENT), SEGMENT_POOL_TAG);
- __LookasideInit(&Pdo->IndirectList, sizeof(XENVBD_INDIRECT), INDIRECT_POOL_TAG);
-
- Status = FrontendCreate(Pdo, DeviceId, TargetId, &Pdo->Frontend);
- if (!NT_SUCCESS(Status))
- goto fail2;
-
- Status = PdoD3ToD0(Pdo);
- if (!NT_SUCCESS(Status))
- goto fail3;
-
- if (!AdapterLinkPdo(Adapter, Pdo))
- goto fail4;
-
- Verbose("Target[%d] : Created (%s)\n", TargetId, Pdo);
- Trace("Target[%d] @ (%d) <=====\n", TargetId, KeGetCurrentIrql());
- return TRUE;
-
-fail4:
- Error("Fail4\n");
- PdoD0ToD3(Pdo);
-
-fail3:
- Error("Fail3\n");
- FrontendDestroy(Pdo->Frontend);
- Pdo->Frontend = NULL;
-
-fail2:
- Error("Fail2\n");
- __LookasideTerm(&Pdo->IndirectList);
- __LookasideTerm(&Pdo->SegmentList);
- __LookasideTerm(&Pdo->RequestList);
- __PdoFree(Pdo);
-
-fail1:
- Error("Fail1 (%08x)\n", Status);
- return FALSE;
-}
-
-VOID
-PdoDestroy(
- __in PXENVBD_PDO Pdo
- )
-{
- const ULONG TargetId = PdoGetTargetId(Pdo);
- PVOID Objects[4];
- PKWAIT_BLOCK WaitBlock;
-
- Trace("Target[%d] @ (%d) =====>\n", TargetId, KeGetCurrentIrql());
- Verbose("Target[%d] : Destroying\n", TargetId);
-
- ASSERT3U(Pdo->Signature, ==, PDO_SIGNATURE);
- if (!AdapterUnlinkPdo(PdoGetAdapter(Pdo), Pdo)) {
- Error("Target[%d] : PDO 0x%p not linked to ADAPTER 0x%p\n", TargetId, Pdo, PdoGetAdapter(Pdo));
- }
-
- PdoD0ToD3(Pdo);
- PdoDereference(Pdo); // drop initial ref count
-
- // Wait for ReferenceCount == 0 and RequestListUsed == 0
- Verbose("Target[%d] : ReferenceCount %d, RequestListUsed %d\n", TargetId, Pdo->ReferenceCount, Pdo->RequestList.Used);
- Objects[0] = &Pdo->RemoveEvent;
- Objects[1] = &Pdo->RequestList.Empty;
- Objects[2] = &Pdo->SegmentList.Empty;
- Objects[3] = &Pdo->IndirectList.Empty;
-
- WaitBlock = (PKWAIT_BLOCK)__PdoAlloc(sizeof(KWAIT_BLOCK) * ARRAYSIZE(Objects));
- if (WaitBlock == NULL) {
- ULONG Index;
-
- Error("Unable to allocate resources for KWAIT_BLOCK\n");
-
- for (Index = 0; Index < ARRAYSIZE(Objects); Index++)
- KeWaitForSingleObject(Objects[Index],
- Executive,
- KernelMode,
- FALSE,
- NULL);
- } else {
- KeWaitForMultipleObjects(ARRAYSIZE(Objects),
- Objects,
- WaitAll,
- Executive,
- KernelMode,
- FALSE,
- NULL,
- WaitBlock);
-#pragma prefast(suppress:6102)
- __PdoFree(WaitBlock);
- }
-
- ASSERT3S(Pdo->ReferenceCount, ==, 0);
- ASSERT3U(PdoGetDevicePnpState(Pdo), ==, Deleted);
-
- FrontendDestroy(Pdo->Frontend);
- Pdo->Frontend = NULL;
-
- __LookasideTerm(&Pdo->IndirectList);
- __LookasideTerm(&Pdo->SegmentList);
- __LookasideTerm(&Pdo->RequestList);
-
- ASSERT3U(Pdo->Signature, ==, PDO_SIGNATURE);
- RtlZeroMemory(Pdo, sizeof(XENVBD_PDO));
- __PdoFree(Pdo);
-
- Verbose("Target[%d] : Destroyed\n", TargetId);
- Trace("Target[%d] @ (%d) <=====\n", TargetId, KeGetCurrentIrql());
-}
+++ /dev/null
-/* Copyright (c) Citrix Systems Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms,
- * with or without modification, are permitted provided
- * that the following conditions are met:
- *
- * * Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the
- * following disclaimer.
- * * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the
- * following disclaimer in the documentation and/or other
- * materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef _XENVBD_PDO_H
-#define _XENVBD_PDO_H
-
-typedef struct _XENVBD_PDO XENVBD_PDO, *PXENVBD_PDO;
-
-#include <ntddk.h>
-#include <ntstrsafe.h>
-#include <xenvbd-storport.h>
-#include "adapter.h"
-#include "srbext.h"
-#include "types.h"
-#include <debug_interface.h>
-
-extern VOID
-PdoDebugCallback(
- __in PXENVBD_PDO Pdo,
- __in PXENBUS_DEBUG_INTERFACE Debug
- );
-
-// Creation/Deletion
-__checkReturn
-extern BOOLEAN
-PdoCreate(
- __in PXENVBD_ADAPTER Adapter,
- __in __nullterminated PCHAR DeviceId,
- __in ULONG TargetId,
- __in XENVBD_DEVICE_TYPE DeviceType
- );
-
-extern VOID
-PdoDestroy(
- __in PXENVBD_PDO Pdo
- );
-
-__checkReturn
-extern NTSTATUS
-PdoD3ToD0(
- __in PXENVBD_PDO Pdo
- );
-
-extern VOID
-PdoD0ToD3(
- __in PXENVBD_PDO Pdo
- );
-
-// PnP States
-extern VOID
-PdoSetMissing(
- __in PXENVBD_PDO Pdo,
- __in __nullterminated const CHAR* Reason
- );
-
-__checkReturn
-extern BOOLEAN
-PdoIsMissing(
- __in PXENVBD_PDO Pdo
- );
-
-extern const CHAR*
-PdoMissingReason(
- __in PXENVBD_PDO Pdo
- );
-
-__checkReturn
-extern BOOLEAN
-PdoIsEmulatedUnplugged(
- __in PXENVBD_PDO Pdo
- );
-
-extern VOID
-PdoSetDevicePnpState(
- __in PXENVBD_PDO Pdo,
- __in DEVICE_PNP_STATE State
- );
-
-__checkReturn
-extern DEVICE_PNP_STATE
-PdoGetDevicePnpState(
- __in PXENVBD_PDO Pdo
- );
-
-// Reference Counting
-extern LONG
-__PdoReference(
- __in PXENVBD_PDO Pdo,
- __in PCHAR Caller
- );
-
-#define PdoReference(_x_) __PdoReference(_x_, __FUNCTION__)
-
-extern LONG
-__PdoDereference(
- __in PXENVBD_PDO Pdo,
- __in PCHAR Caller
- );
-
-#define PdoDereference(_x_) __PdoDereference(_x_, __FUNCTION__)
-
-// Query Methods
-extern ULONG
-PdoGetTargetId(
- __in PXENVBD_PDO Pdo
- );
-
-__checkReturn
-extern PDEVICE_OBJECT
-PdoGetDeviceObject(
- __in PXENVBD_PDO Pdo
- );
-
-extern VOID
-PdoSetDeviceObject(
- __in PXENVBD_PDO Pdo,
- __in PDEVICE_OBJECT DeviceObject
- );
-
-__checkReturn
-extern BOOLEAN
-PdoIsPaused(
- __in PXENVBD_PDO Pdo
- );
-
-__checkReturn
-extern ULONG
-PdoOutstandingReqs(
- __in PXENVBD_PDO Pdo
- );
-
-__checkReturn
-extern PXENVBD_ADAPTER
-PdoGetAdapter(
- __in PXENVBD_PDO Pdo
- );
-
-extern ULONG
-PdoSectorSize(
- __in PXENVBD_PDO Pdo
- );
-
-// Queue-Related
-extern VOID
-PdoSubmitRequests(
- __in PXENVBD_PDO Pdo
- );
-
-extern VOID
-PdoCompleteResponse(
- __in PXENVBD_PDO Pdo,
- __in ULONG Tag,
- __in SHORT Status
- );
-
-extern VOID
-PdoPreResume(
- __in PXENVBD_PDO Pdo
- );
-
-extern VOID
-PdoPostResume(
- __in PXENVBD_PDO Pdo
- );
-
-// StorPort Methods
-extern VOID
-PdoReset(
- __in PXENVBD_PDO Pdo
- );
-
-__checkReturn
-extern BOOLEAN
-PdoStartIo(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_REQUEST_BLOCK Srb
- );
-
-extern VOID
-PdoSrbPnp(
- __in PXENVBD_PDO Pdo,
- __in PSCSI_PNP_REQUEST_BLOCK Srb
- );
-
-// PnP Handler
-__checkReturn
-extern NTSTATUS
-PdoDispatchPnp(
- __in PXENVBD_PDO Pdo,
- __in PDEVICE_OBJECT DeviceObject,
- __in PIRP Irp
- );
-
-__drv_maxIRQL(DISPATCH_LEVEL)
-extern VOID
-PdoIssueDeviceEject(
- __in PXENVBD_PDO Pdo,
- __in __nullterminated const CHAR* Reason
- );
-
-#endif // _XENVBD_PDO_H
--- /dev/null
+/* Copyright (c) Citrix Systems Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms,
+ * with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer in the documentation and/or other
+ * materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "target.h"
+#include "driver.h"
+#include "adapter.h"
+#include "frontend.h"
+#include "queue.h"
+#include "srbext.h"
+#include "buffer.h"
+#include "pdoinquiry.h"
+#include "debug.h"
+#include "assert.h"
+#include "util.h"
+#include <xencdb.h>
+#include <names.h>
+#include <store_interface.h>
+#include <evtchn_interface.h>
+#include <gnttab_interface.h>
+#include <debug_interface.h>
+#include <suspend_interface.h>
+
+typedef struct _XENVBD_SG_LIST {
+ // SGList from SRB
+ PSTOR_SCATTER_GATHER_LIST SGList;
+ // "current" values
+ STOR_PHYSICAL_ADDRESS PhysAddr;
+ ULONG PhysLen;
+ // iteration
+ ULONG Index;
+ ULONG Offset;
+ ULONG Length;
+} XENVBD_SG_LIST, *PXENVBD_SG_LIST;
+
+#define TARGET_SIGNATURE 'odpX'
+
+typedef struct _XENVBD_LOOKASIDE {
+ KEVENT Empty;
+ LONG Used;
+ LONG Max;
+ ULONG Failed;
+ ULONG Size;
+ NPAGED_LOOKASIDE_LIST List;
+} XENVBD_LOOKASIDE, *PXENVBD_LOOKASIDE;
+
+struct _XENVBD_TARGET {
+ ULONG Signature;
+ PXENVBD_ADAPTER Adapter;
+ PDEVICE_OBJECT DeviceObject;
+ KEVENT RemoveEvent;
+ LONG ReferenceCount;
+ DEVICE_PNP_STATE DevicePnpState;
+ DEVICE_PNP_STATE PrevPnpState;
+ DEVICE_POWER_STATE DevicePowerState;
+ KSPIN_LOCK Lock;
+
+ // Frontend (Ring, includes XenBus interfaces)
+ PXENVBD_FRONTEND Frontend;
+ XENVBD_DEVICE_TYPE DeviceType;
+
+ // State
+ LONG Paused;
+
+ // Eject
+ BOOLEAN WrittenEjected;
+ BOOLEAN EjectRequested;
+ BOOLEAN EjectPending;
+ BOOLEAN Missing;
+ const CHAR* Reason;
+
+ // SRBs
+ XENVBD_LOOKASIDE RequestList;
+ XENVBD_LOOKASIDE SegmentList;
+ XENVBD_LOOKASIDE IndirectList;
+ XENVBD_QUEUE FreshSrbs;
+ XENVBD_QUEUE PreparedReqs;
+ XENVBD_QUEUE SubmittedReqs;
+ XENVBD_QUEUE ShutdownSrbs;
+ ULONG NextTag;
+
+ // Stats - SRB Counts by BLKIF_OP_
+ ULONG BlkOpRead;
+ ULONG BlkOpWrite;
+ ULONG BlkOpIndirectRead;
+ ULONG BlkOpIndirectWrite;
+ ULONG BlkOpBarrier;
+ ULONG BlkOpDiscard;
+ ULONG BlkOpFlush;
+ // Stats - Failures
+ ULONG FailedMaps;
+ ULONG FailedBounces;
+ ULONG FailedGrants;
+ // Stats - Segments
+ ULONG64 SegsGranted;
+ ULONG64 SegsBounced;
+};
+
+//=============================================================================
+#define TARGET_POOL_TAG 'odPX'
+#define REQUEST_POOL_TAG 'qeRX'
+#define SEGMENT_POOL_TAG 'geSX'
+#define INDIRECT_POOL_TAG 'dnIX'
+
+__checkReturn
+__drv_allocatesMem(mem)
+__bcount(Size)
+static FORCEINLINE PVOID
+#pragma warning(suppress: 28195)
+__TargetAlloc(
+ __in ULONG Size
+ )
+{
+ return __AllocatePoolWithTag(NonPagedPool, Size, TARGET_POOL_TAG);
+}
+
+static FORCEINLINE VOID
+#pragma warning(suppress: 28197)
+__TargetFree(
+ __in __drv_freesMem(mem) PVOID Buffer
+ )
+{
+ if (Buffer)
+ __FreePoolWithTag(Buffer, TARGET_POOL_TAG);
+}
+
+//=============================================================================
+// Lookasides
+static FORCEINLINE VOID
+__LookasideInit(
+ IN OUT PXENVBD_LOOKASIDE Lookaside,
+ IN ULONG Size,
+ IN ULONG Tag
+ )
+{
+ RtlZeroMemory(Lookaside, sizeof(XENVBD_LOOKASIDE));
+ Lookaside->Size = Size;
+ KeInitializeEvent(&Lookaside->Empty, SynchronizationEvent, TRUE);
+ ExInitializeNPagedLookasideList(&Lookaside->List, NULL, NULL, 0,
+ Size, Tag, 0);
+}
+
+static FORCEINLINE VOID
+__LookasideTerm(
+ IN PXENVBD_LOOKASIDE Lookaside
+ )
+{
+ ASSERT3U(Lookaside->Used, ==, 0);
+ ExDeleteNPagedLookasideList(&Lookaside->List);
+ RtlZeroMemory(Lookaside, sizeof(XENVBD_LOOKASIDE));
+}
+
+static FORCEINLINE PVOID
+__LookasideAlloc(
+ IN PXENVBD_LOOKASIDE Lookaside
+ )
+{
+ LONG Result;
+ PVOID Buffer;
+
+ Buffer = ExAllocateFromNPagedLookasideList(&Lookaside->List);
+ if (Buffer == NULL) {
+ ++Lookaside->Failed;
+ return NULL;
+ }
+
+ RtlZeroMemory(Buffer, Lookaside->Size);
+ Result = InterlockedIncrement(&Lookaside->Used);
+ ASSERT3S(Result, >, 0);
+ if (Result > Lookaside->Max)
+ Lookaside->Max = Result;
+ KeClearEvent(&Lookaside->Empty);
+
+ return Buffer;
+}
+
+static FORCEINLINE VOID
+__LookasideFree(
+ IN PXENVBD_LOOKASIDE Lookaside,
+ IN PVOID Buffer
+ )
+{
+ LONG Result;
+
+ ExFreeToNPagedLookasideList(&Lookaside->List, Buffer);
+ Result = InterlockedDecrement(&Lookaside->Used);
+ ASSERT3S(Result, >=, 0);
+
+ if (Result == 0) {
+ KeSetEvent(&Lookaside->Empty, IO_NO_INCREMENT, FALSE);
+ }
+}
+
+static FORCEINLINE VOID
+__LookasideDebug(
+ IN PXENVBD_LOOKASIDE Lookaside,
+ IN PXENBUS_DEBUG_INTERFACE Debug,
+ IN PCHAR Name
+ )
+{
+ XENBUS_DEBUG(Printf, Debug,
+ "LOOKASIDE: %s: %u / %u (%u failed)\n",
+ Name, Lookaside->Used,
+ Lookaside->Max, Lookaside->Failed);
+
+ Lookaside->Max = Lookaside->Used;
+ Lookaside->Failed = 0;
+}
+
+//=============================================================================
+// Debug
+static FORCEINLINE PCHAR
+__PnpStateName(
+ __in DEVICE_PNP_STATE State
+ )
+{
+ switch (State) {
+ case Invalid: return "Invalid";
+ case Present: return "Present";
+ case Enumerated: return "Enumerated";
+ case Added: return "Added";
+ case Started: return "Started";
+ case StopPending: return "StopPending";
+ case Stopped: return "Stopped";
+ case RemovePending: return "RemovePending";
+ case SurpriseRemovePending: return "SurpriseRemovePending";
+ case Deleted: return "Deleted";
+ default: return "UNKNOWN";
+ }
+}
+
+DECLSPEC_NOINLINE VOID
+TargetDebugCallback(
+ __in PXENVBD_TARGET Target,
+ __in PXENBUS_DEBUG_INTERFACE DebugInterface
+ )
+{
+ if (Target == NULL || DebugInterface == NULL)
+ return;
+ if (Target->Signature != TARGET_SIGNATURE)
+ return;
+
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: Adapter 0x%p DeviceObject 0x%p\n",
+ Target->Adapter,
+ Target->DeviceObject);
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: ReferenceCount %d\n",
+ Target->ReferenceCount);
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: DevicePnpState %s (%s)\n",
+ __PnpStateName(Target->DevicePnpState),
+ __PnpStateName(Target->PrevPnpState));
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: DevicePowerState %s\n",
+ PowerDeviceStateName(Target->DevicePowerState));
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: %s\n",
+ Target->Missing ? Target->Reason : "Not Missing");
+
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: BLKIF_OPs: READ=%u WRITE=%u\n",
+ Target->BlkOpRead, Target->BlkOpWrite);
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: BLKIF_OPs: INDIRECT_READ=%u INDIRECT_WRITE=%u\n",
+ Target->BlkOpIndirectRead, Target->BlkOpIndirectWrite);
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: BLKIF_OPs: BARRIER=%u DISCARD=%u FLUSH=%u\n",
+ Target->BlkOpBarrier, Target->BlkOpDiscard, Target->BlkOpFlush);
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: Failed: Maps=%u Bounces=%u Grants=%u\n",
+ Target->FailedMaps, Target->FailedBounces, Target->FailedGrants);
+ XENBUS_DEBUG(Printf, DebugInterface,
+ "TARGET: Segments Granted=%llu Bounced=%llu\n",
+ Target->SegsGranted, Target->SegsBounced);
+
+ __LookasideDebug(&Target->RequestList, DebugInterface, "REQUESTs");
+ __LookasideDebug(&Target->SegmentList, DebugInterface, "SEGMENTs");
+ __LookasideDebug(&Target->IndirectList, DebugInterface, "INDIRECTs");
+
+ QueueDebugCallback(&Target->FreshSrbs, "Fresh ", DebugInterface);
+ QueueDebugCallback(&Target->PreparedReqs, "Prepared ", DebugInterface);
+ QueueDebugCallback(&Target->SubmittedReqs, "Submitted", DebugInterface);
+ QueueDebugCallback(&Target->ShutdownSrbs, "Shutdown ", DebugInterface);
+
+ FrontendDebugCallback(Target->Frontend, DebugInterface);
+
+ Target->BlkOpRead = Target->BlkOpWrite = 0;
+ Target->BlkOpIndirectRead = Target->BlkOpIndirectWrite = 0;
+ Target->BlkOpBarrier = Target->BlkOpDiscard = Target->BlkOpFlush = 0;
+ Target->FailedMaps = Target->FailedBounces = Target->FailedGrants = 0;
+ Target->SegsGranted = Target->SegsBounced = 0;
+}
+
+//=============================================================================
+// Power States
+__checkReturn
+static FORCEINLINE BOOLEAN
+TargetSetDevicePowerState(
+ __in PXENVBD_TARGET Target,
+ __in DEVICE_POWER_STATE State
+ )
+{
+ KIRQL Irql;
+ BOOLEAN Changed = FALSE;
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ if (Target->DevicePowerState != State) {
+ Verbose("Target[%d] : POWER %s to %s\n", TargetGetTargetId(Target), PowerDeviceStateName(Target->DevicePowerState), PowerDeviceStateName(State));
+ Target->DevicePowerState = State;
+ Changed = TRUE;
+ }
+ KeReleaseSpinLock(&Target->Lock, Irql);
+
+ return Changed;
+}
+
+//=============================================================================
+// PnP States
+FORCEINLINE VOID
+TargetSetMissing(
+ __in PXENVBD_TARGET Target,
+ __in __nullterminated const CHAR* Reason
+ )
+{
+ KIRQL Irql;
+
+ ASSERT3P(Reason, !=, NULL);
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ if (Target->Missing) {
+ Verbose("Target[%d] : Already MISSING (%s) when trying to set (%s)\n", TargetGetTargetId(Target), Target->Reason, Reason);
+ } else {
+ Verbose("Target[%d] : MISSING %s\n", TargetGetTargetId(Target), Reason);
+ Target->Missing = TRUE;
+ Target->Reason = Reason;
+ }
+ KeReleaseSpinLock(&Target->Lock, Irql);
+}
+
+__checkReturn
+FORCEINLINE BOOLEAN
+TargetIsMissing(
+ __in PXENVBD_TARGET Target
+ )
+{
+ KIRQL Irql;
+ BOOLEAN Missing;
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ Missing = Target->Missing;
+ KeReleaseSpinLock(&Target->Lock, Irql);
+
+ return Missing;
+}
+
+FORCEINLINE const CHAR*
+TargetMissingReason(
+ __in PXENVBD_TARGET Target
+ )
+{
+ KIRQL Irql;
+ const CHAR* Reason;
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ Reason = Target->Reason;
+ KeReleaseSpinLock(&Target->Lock, Irql);
+
+ return Reason;
+}
+
+FORCEINLINE VOID
+TargetSetDevicePnpState(
+ __in PXENVBD_TARGET Target,
+ __in DEVICE_PNP_STATE State
+ )
+{
+ Verbose("Target[%d] : PNP %s to %s\n",
+ TargetGetTargetId(Target),
+ __PnpStateName(Target->DevicePnpState),
+ __PnpStateName(State));
+
+ if (Target->DevicePnpState == Deleted)
+ return;
+
+ Target->PrevPnpState = Target->DevicePnpState;
+ Target->DevicePnpState = State;
+}
+
+__checkReturn
+FORCEINLINE DEVICE_PNP_STATE
+TargetGetDevicePnpState(
+ __in PXENVBD_TARGET Target
+ )
+{
+ return Target->DevicePnpState;
+}
+
+static FORCEINLINE VOID
+__TargetRestoreDevicePnpState(
+ __in PXENVBD_TARGET Target,
+ __in DEVICE_PNP_STATE State
+ )
+{
+ if (Target->DevicePnpState == State) {
+ Verbose("Target[%d] : PNP %s to %s\n", TargetGetTargetId(Target), __PnpStateName(Target->DevicePnpState), __PnpStateName(Target->PrevPnpState));
+ Target->DevicePnpState = Target->PrevPnpState;
+ }
+}
+
+//=============================================================================
+// Reference Counting
+FORCEINLINE LONG
+__TargetReference(
+ __in PXENVBD_TARGET Target,
+ __in PCHAR Caller
+ )
+{
+ LONG Result;
+
+ ASSERT3P(Target, !=, NULL);
+ Result = InterlockedIncrement(&Target->ReferenceCount);
+ ASSERTREFCOUNT(Result, >, 0, Caller);
+
+ if (Result == 1) {
+ Result = InterlockedDecrement(&Target->ReferenceCount);
+ Error("Target[%d] : %s: Attempting to take reference of removed TARGET from %d\n", TargetGetTargetId(Target), Caller, Result);
+ return 0;
+ } else {
+ ASSERTREFCOUNT(Result, >, 1, Caller);
+ return Result;
+ }
+}
+
+FORCEINLINE LONG
+__TargetDereference(
+ __in PXENVBD_TARGET Target,
+ __in PCHAR Caller
+ )
+{
+ LONG Result;
+
+ ASSERT3P(Target, !=, NULL);
+ Result = InterlockedDecrement(&Target->ReferenceCount);
+ ASSERTREFCOUNT(Result, >=, 0, Caller);
+
+ if (Result == 0) {
+ Verbose("Final ReferenceCount dropped, Target[%d] able to be removed\n", TargetGetTargetId(Target));
+ KeSetEvent(&Target->RemoveEvent, IO_NO_INCREMENT, FALSE);
+ }
+ return Result;
+}
+
+//=============================================================================
+// Query Methods
+FORCEINLINE ULONG
+TargetGetTargetId(
+ __in PXENVBD_TARGET Target
+ )
+{
+ ASSERT3P(Target, !=, NULL);
+ return FrontendGetTargetId(Target->Frontend);
+}
+
+__checkReturn
+FORCEINLINE PDEVICE_OBJECT
+TargetGetDeviceObject(
+ __in PXENVBD_TARGET Target
+ )
+{
+ ASSERT3P(Target, !=, NULL);
+ return Target->DeviceObject;
+}
+
+FORCEINLINE VOID
+TargetSetDeviceObject(
+ __in PXENVBD_TARGET Target,
+ __in PDEVICE_OBJECT DeviceObject
+ )
+{
+ Verbose("Target[%d] : Setting DeviceObject = 0x%p\n", TargetGetTargetId(Target), DeviceObject);
+
+ ASSERT3P(Target->DeviceObject, ==, NULL);
+ Target->DeviceObject = DeviceObject;
+}
+
+__checkReturn
+FORCEINLINE BOOLEAN
+TargetIsPaused(
+ __in PXENVBD_TARGET Target
+ )
+{
+ BOOLEAN Paused;
+ KIRQL Irql;
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ Paused = (Target->Paused > 0);
+ KeReleaseSpinLock(&Target->Lock, Irql);
+
+ return Paused;
+}
+
+__checkReturn
+FORCEINLINE ULONG
+TargetOutstandingReqs(
+ __in PXENVBD_TARGET Target
+ )
+{
+ return QueueCount(&Target->SubmittedReqs);
+}
+
+__checkReturn
+FORCEINLINE PXENVBD_ADAPTER
+TargetGetAdapter(
+ __in PXENVBD_TARGET Target
+ )
+{
+ return Target->Adapter;
+}
+
+FORCEINLINE ULONG
+TargetSectorSize(
+ __in PXENVBD_TARGET Target
+ )
+{
+ return FrontendGetDiskInfo(Target->Frontend)->SectorSize;
+}
+
+//=============================================================================
+static PXENVBD_INDIRECT
+TargetGetIndirect(
+ IN PXENVBD_TARGET Target
+ )
+{
+ PXENVBD_INDIRECT Indirect;
+ NTSTATUS status;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Target->Frontend);
+
+ Indirect = __LookasideAlloc(&Target->IndirectList);
+ if (Indirect == NULL)
+ goto fail1;
+
+ RtlZeroMemory(Indirect, sizeof(XENVBD_INDIRECT));
+
+ Indirect->Mdl = __AllocatePage();
+ if (Indirect->Mdl == NULL)
+ goto fail2;
+
+ Indirect->Page = MmGetSystemAddressForMdlSafe(Indirect->Mdl,
+ NormalPagePriority);
+
+ status = GranterGet(Granter,
+ MmGetMdlPfnArray(Indirect->Mdl)[0],
+ TRUE,
+ &Indirect->Grant);
+ if (!NT_SUCCESS(status))
+ goto fail3;
+
+ return Indirect;
+
+fail3:
+ __FreePage(Indirect->Mdl);
+fail2:
+ __LookasideFree(&Target->IndirectList, Indirect);
+fail1:
+ return NULL;
+}
+
+static VOID
+TargetPutIndirect(
+ IN PXENVBD_TARGET Target,
+ IN PXENVBD_INDIRECT Indirect
+ )
+{
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Target->Frontend);
+
+ if (Indirect->Grant)
+ GranterPut(Granter, Indirect->Grant);
+ if (Indirect->Page)
+ __FreePage(Indirect->Mdl);
+
+ RtlZeroMemory(Indirect, sizeof(XENVBD_INDIRECT));
+ __LookasideFree(&Target->IndirectList, Indirect);
+}
+
+static PXENVBD_SEGMENT
+TargetGetSegment(
+ IN PXENVBD_TARGET Target
+ )
+{
+ PXENVBD_SEGMENT Segment;
+
+ Segment = __LookasideAlloc(&Target->SegmentList);
+ if (Segment == NULL)
+ goto fail1;
+
+ RtlZeroMemory(Segment, sizeof(XENVBD_SEGMENT));
+ return Segment;
+
+fail1:
+ return NULL;
+}
+
+static VOID
+TargetPutSegment(
+ IN PXENVBD_TARGET Target,
+ IN PXENVBD_SEGMENT Segment
+ )
+{
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Target->Frontend);
+
+ if (Segment->Grant)
+ GranterPut(Granter, Segment->Grant);
+
+ if (Segment->BufferId)
+ BufferPut(Segment->BufferId);
+
+ if (Segment->Buffer)
+ MmUnmapLockedPages(Segment->Buffer, &Segment->Mdl);
+
+ RtlZeroMemory(Segment, sizeof(XENVBD_SEGMENT));
+ __LookasideFree(&Target->SegmentList, Segment);
+}
+
+static PXENVBD_REQUEST
+TargetGetRequest(
+ IN PXENVBD_TARGET Target
+ )
+{
+ PXENVBD_REQUEST Request;
+
+ Request = __LookasideAlloc(&Target->RequestList);
+ if (Request == NULL)
+ goto fail1;
+
+ RtlZeroMemory(Request, sizeof(XENVBD_REQUEST));
+ Request->Id = (ULONG)InterlockedIncrement((PLONG)&Target->NextTag);
+ InitializeListHead(&Request->Segments);
+ InitializeListHead(&Request->Indirects);
+
+ return Request;
+
+fail1:
+ return NULL;
+}
+
+static VOID
+TargetPutRequest(
+ IN PXENVBD_TARGET Target,
+ IN PXENVBD_REQUEST Request
+ )
+{
+ PLIST_ENTRY Entry;
+
+ for (;;) {
+ PXENVBD_SEGMENT Segment;
+
+ Entry = RemoveHeadList(&Request->Segments);
+ if (Entry == &Request->Segments)
+ break;
+ Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry);
+ TargetPutSegment(Target, Segment);
+ }
+
+ for (;;) {
+ PXENVBD_INDIRECT Indirect;
+
+ Entry = RemoveHeadList(&Request->Indirects);
+ if (Entry == &Request->Indirects)
+ break;
+ Indirect = CONTAINING_RECORD(Entry, XENVBD_INDIRECT, Entry);
+ TargetPutIndirect(Target, Indirect);
+ }
+
+ RtlZeroMemory(Request, sizeof(XENVBD_REQUEST));
+ __LookasideFree(&Target->RequestList, Request);
+}
+
+static FORCEINLINE PXENVBD_REQUEST
+TargetRequestFromTag(
+ IN PXENVBD_TARGET Target,
+ IN ULONG Tag
+ )
+{
+ KIRQL Irql;
+ PLIST_ENTRY Entry;
+ PXENVBD_QUEUE Queue = &Target->SubmittedReqs;
+
+ KeAcquireSpinLock(&Queue->Lock, &Irql);
+
+ for (Entry = Queue->List.Flink; Entry != &Queue->List; Entry = Entry->Flink) {
+ PXENVBD_REQUEST Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+ if (Request->Id == Tag) {
+ RemoveEntryList(&Request->Entry);
+ --Queue->Current;
+ KeReleaseSpinLock(&Queue->Lock, Irql);
+ return Request;
+ }
+ }
+
+ KeReleaseSpinLock(&Queue->Lock, Irql);
+ Warning("Target[%d] : Tag %x not found in submitted list (%u items)\n",
+ TargetGetTargetId(Target), Tag, QueueCount(Queue));
+ return NULL;
+}
+
+static FORCEINLINE VOID
+__TargetIncBlkifOpCount(
+ __in PXENVBD_TARGET Target,
+ __in PXENVBD_REQUEST Request
+ )
+{
+ switch (Request->Operation) {
+ case BLKIF_OP_READ:
+ if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
+ ++Target->BlkOpIndirectRead;
+ else
+ ++Target->BlkOpRead;
+ break;
+ case BLKIF_OP_WRITE:
+ if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST)
+ ++Target->BlkOpIndirectWrite;
+ else
+ ++Target->BlkOpWrite;
+ break;
+ case BLKIF_OP_WRITE_BARRIER:
+ ++Target->BlkOpBarrier;
+ break;
+ case BLKIF_OP_DISCARD:
+ ++Target->BlkOpDiscard;
+ break;
+ case BLKIF_OP_FLUSH_DISKCACHE:
+ ++Target->BlkOpFlush;
+ break;
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+}
+
+static FORCEINLINE ULONG
+__SectorsPerPage(
+ __in ULONG SectorSize
+ )
+{
+ ASSERT3U(SectorSize, !=, 0);
+ return PAGE_SIZE / SectorSize;
+}
+
+static FORCEINLINE VOID
+__Operation(
+ __in UCHAR CdbOp,
+ __out PUCHAR RingOp,
+ __out PBOOLEAN ReadOnly
+ )
+{
+ switch (CdbOp) {
+ case SCSIOP_READ:
+ *RingOp = BLKIF_OP_READ;
+ *ReadOnly = FALSE;
+ break;
+ case SCSIOP_WRITE:
+ *RingOp = BLKIF_OP_WRITE;
+ *ReadOnly = TRUE;
+ break;
+ default:
+ ASSERT(FALSE);
+ }
+}
+
+static FORCEINLINE ULONG
+__Offset(
+ __in STOR_PHYSICAL_ADDRESS PhysAddr
+ )
+{
+ return (ULONG)(PhysAddr.QuadPart & (PAGE_SIZE - 1));
+}
+
+static FORCEINLINE PFN_NUMBER
+__Phys2Pfn(
+ __in STOR_PHYSICAL_ADDRESS PhysAddr
+ )
+{
+ return (PFN_NUMBER)(PhysAddr.QuadPart >> PAGE_SHIFT);
+}
+
+static FORCEINLINE PFN_NUMBER
+__Virt2Pfn(
+ __in PVOID VirtAddr
+ )
+{
+ return (PFN_NUMBER)(MmGetPhysicalAddress(VirtAddr).QuadPart >> PAGE_SHIFT);
+}
+
+static FORCEINLINE MM_PAGE_PRIORITY
+__TargetPriority(
+ __in PXENVBD_TARGET Target
+ )
+{
+ PXENVBD_CAPS Caps = FrontendGetCaps(Target->Frontend);
+ if (!(Caps->Paging ||
+ Caps->Hibernation ||
+ Caps->DumpFile))
+ return NormalPagePriority;
+
+ return HighPagePriority;
+}
+
+#define __min(_x, _y) ((_x) < (_y)) ? (_x) : (_y)
+
+static FORCEINLINE VOID
+SGListGet(
+ IN OUT PXENVBD_SG_LIST SGList
+ )
+{
+ PSTOR_SCATTER_GATHER_ELEMENT SGElement;
+
+ ASSERT3U(SGList->Index, <, SGList->SGList->NumberOfElements);
+
+ SGElement = &SGList->SGList->List[SGList->Index];
+
+ SGList->PhysAddr.QuadPart = SGElement->PhysicalAddress.QuadPart + SGList->Offset;
+ SGList->PhysLen = __min(PAGE_SIZE - __Offset(SGList->PhysAddr) - SGList->Length, SGElement->Length - SGList->Offset);
+
+ ASSERT3U(SGList->PhysLen, <=, PAGE_SIZE);
+ ASSERT3U(SGList->Offset, <, SGElement->Length);
+
+ SGList->Length = SGList->PhysLen; // gets reset every time for Granted, every 1or2 times for Bounced
+ SGList->Offset = SGList->Offset + SGList->PhysLen;
+ if (SGList->Offset >= SGElement->Length) {
+ SGList->Index = SGList->Index + 1;
+ SGList->Offset = 0;
+ }
+}
+
+static FORCEINLINE BOOLEAN
+SGListNext(
+ IN OUT PXENVBD_SG_LIST SGList,
+ IN ULONG AlignmentMask
+ )
+{
+ SGList->Length = 0;
+ SGListGet(SGList); // get next PhysAddr and PhysLen
+ return !((SGList->PhysAddr.QuadPart & AlignmentMask) || (SGList->PhysLen & AlignmentMask));
+}
+
+static FORCEINLINE BOOLEAN
+MapSegmentBuffer(
+ IN PXENVBD_TARGET Target,
+ IN PXENVBD_SEGMENT Segment,
+ IN PXENVBD_SG_LIST SGList,
+ IN ULONG SectorSize,
+ IN ULONG SectorsNow
+ )
+{
+ PMDL Mdl;
+
+ // map PhysAddr to 1 or 2 pages and lock for VirtAddr
+#pragma warning(push)
+#pragma warning(disable:28145)
+ Mdl = &Segment->Mdl;
+ Mdl->Next = NULL;
+ Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER));
+ Mdl->MdlFlags = MDL_PAGES_LOCKED;
+ Mdl->Process = NULL;
+ Mdl->MappedSystemVa = NULL;
+ Mdl->StartVa = NULL;
+ Mdl->ByteCount = SGList->PhysLen;
+ Mdl->ByteOffset = __Offset(SGList->PhysAddr);
+ Segment->Pfn[0] = __Phys2Pfn(SGList->PhysAddr);
+
+ if (SGList->PhysLen < SectorsNow * SectorSize) {
+ SGListGet(SGList);
+ Mdl->Size += sizeof(PFN_NUMBER);
+ Mdl->ByteCount = Mdl->ByteCount + SGList->PhysLen;
+ Segment->Pfn[1] = __Phys2Pfn(SGList->PhysAddr);
+ }
+#pragma warning(pop)
+
+ ASSERT((Mdl->ByteCount & (SectorSize - 1)) == 0);
+ ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE);
+ ASSERT3U(SectorsNow, ==, (Mdl->ByteCount / SectorSize));
+
+ Segment->Length = __min(Mdl->ByteCount, PAGE_SIZE);
+ Segment->Buffer = MmMapLockedPagesSpecifyCache(Mdl, KernelMode,
+ MmCached, NULL, FALSE, __TargetPriority(Target));
+ if (!Segment->Buffer) {
+ goto fail;
+ }
+
+ ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Segment->Pfn[0]);
+ ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Segment->Pfn[1]);
+
+ return TRUE;
+
+fail:
+ return FALSE;
+}
+
+static FORCEINLINE VOID
+RequestCopyOutput(
+ __in PXENVBD_REQUEST Request
+ )
+{
+ PLIST_ENTRY Entry;
+
+ if (Request->Operation != BLKIF_OP_READ)
+ return;
+
+ for (Entry = Request->Segments.Flink;
+ Entry != &Request->Segments;
+ Entry = Entry->Flink) {
+ PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry);
+
+ if (Segment->BufferId)
+ BufferCopyOut(Segment->BufferId, Segment->Buffer, Segment->Length);
+ }
+}
+
+static BOOLEAN
+PrepareSegment(
+ IN PXENVBD_TARGET Target,
+ IN PXENVBD_SEGMENT Segment,
+ IN PXENVBD_SG_LIST SGList,
+ IN BOOLEAN ReadOnly,
+ IN ULONG SectorsLeft,
+ OUT PULONG SectorsNow
+ )
+{
+ PFN_NUMBER Pfn;
+ NTSTATUS Status;
+ PXENVBD_GRANTER Granter = FrontendGetGranter(Target->Frontend);
+ const ULONG SectorSize = TargetSectorSize(Target);
+ const ULONG SectorsPerPage = __SectorsPerPage(SectorSize);
+
+ if (SGListNext(SGList, SectorSize - 1)) {
+ ++Target->SegsGranted;
+ // get first sector, last sector and count
+ Segment->FirstSector = (UCHAR)((__Offset(SGList->PhysAddr) + SectorSize - 1) / SectorSize);
+ *SectorsNow = __min(SectorsLeft, SectorsPerPage - Segment->FirstSector);
+ Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow - 1);
+ Segment->BufferId = NULL; // granted, ensure its null
+ Segment->Buffer = NULL; // granted, ensure its null
+ Segment->Length = 0; // granted, ensure its 0
+ Pfn = __Phys2Pfn(SGList->PhysAddr);
+
+ ASSERT3U((SGList->PhysLen / SectorSize), ==, *SectorsNow);
+ ASSERT3U((SGList->PhysLen & (SectorSize - 1)), ==, 0);
+ } else {
+ ++Target->SegsBounced;
+ // get first sector, last sector and count
+ Segment->FirstSector = 0;
+ *SectorsNow = __min(SectorsLeft, SectorsPerPage);
+ Segment->LastSector = (UCHAR)(*SectorsNow - 1);
+
+ // map SGList to Virtual Address. Populates Segment->Buffer and Segment->Length
+ if (!MapSegmentBuffer(Target, Segment, SGList, SectorSize, *SectorsNow)) {
+ ++Target->FailedMaps;
+ goto fail1;
+ }
+
+ // get a buffer
+ if (!BufferGet(Segment, &Segment->BufferId, &Pfn)) {
+ ++Target->FailedBounces;
+ goto fail2;
+ }
+
+ // copy contents in
+ if (ReadOnly) { // Operation == BLKIF_OP_WRITE
+ BufferCopyIn(Segment->BufferId, Segment->Buffer, Segment->Length);
+ }
+ }
+
+ // Grant segment's page
+ Status = GranterGet(Granter, Pfn, ReadOnly, &Segment->Grant);
+ if (!NT_SUCCESS(Status)) {
+ ++Target->FailedGrants;
+ goto fail3;
+ }
+
+ return TRUE;
+
+fail3:
+fail2:
+fail1:
+ return FALSE;
+}
+
+static BOOLEAN
+PrepareBlkifReadWrite(
+ IN PXENVBD_TARGET Target,
+ IN PXENVBD_REQUEST Request,
+ IN PXENVBD_SG_LIST SGList,
+ IN ULONG MaxSegments,
+ IN ULONG64 SectorStart,
+ IN ULONG SectorsLeft,
+ OUT PULONG SectorsDone
+ )
+{
+ UCHAR Operation;
+ BOOLEAN ReadOnly;
+ ULONG Index;
+ __Operation(Cdb_OperationEx(Request->Srb), &Operation, &ReadOnly);
+
+ Request->Operation = Operation;
+ Request->NrSegments = 0;
+ Request->FirstSector = SectorStart;
+
+ for (Index = 0;
+ Index < MaxSegments &&
+ SectorsLeft > 0;
+ ++Index) {
+ PXENVBD_SEGMENT Segment;
+ ULONG SectorsNow;
+
+ Segment = TargetGetSegment(Target);
+ if (Segment == NULL)
+ goto fail1;
+
+ InsertTailList(&Request->Segments, &Segment->Entry);
+ ++Request->NrSegments;
+
+ if (!PrepareSegment(Target,
+ Segment,
+ SGList,
+ ReadOnly,
+ SectorsLeft,
+ &SectorsNow))
+ goto fail2;
+
+ *SectorsDone += SectorsNow;
+ SectorsLeft -= SectorsNow;
+ }
+ ASSERT3U(Request->NrSegments, >, 0);
+ ASSERT3U(Request->NrSegments, <=, MaxSegments);
+
+ return TRUE;
+
+fail2:
+fail1:
+ return FALSE;
+}
+
+static BOOLEAN
+PrepareBlkifIndirect(
+ IN PXENVBD_TARGET Target,
+ IN PXENVBD_REQUEST Request
+ )
+{
+ ULONG Index;
+ ULONG NrSegments = 0;
+
+ for (Index = 0;
+ Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST &&
+ NrSegments < Request->NrSegments;
+ ++Index) {
+ PXENVBD_INDIRECT Indirect;
+
+ Indirect = TargetGetIndirect(Target);
+ if (Indirect == NULL)
+ goto fail1;
+ InsertTailList(&Request->Indirects, &Indirect->Entry);
+
+ NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE;
+ }
+
+ return TRUE;
+
+fail1:
+ return FALSE;
+}
+
+static FORCEINLINE ULONG
+UseIndirect(
+ IN PXENVBD_TARGET Target,
+ IN ULONG SectorsLeft
+ )
+{
+ const ULONG SectorsPerPage = __SectorsPerPage(TargetSectorSize(Target));
+ const ULONG MaxIndirectSegs = FrontendGetFeatures(Target->Frontend)->Indirect;
+
+ if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
+ return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported
+
+ if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage)
+ return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single BLKIF_OP_{READ/WRITE}
+
+ return MaxIndirectSegs;
+}
+
+static FORCEINLINE ULONG
+TargetQueueRequestList(
+ IN PXENVBD_TARGET Target,
+ IN PLIST_ENTRY List
+ )
+{
+ ULONG Count = 0;
+ for (;;) {
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY Entry;
+
+ Entry = RemoveHeadList(List);
+ if (Entry == List)
+ break;
+
+ ++Count;
+ Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+ __TargetIncBlkifOpCount(Target, Request);
+ QueueAppend(&Target->PreparedReqs, &Request->Entry);
+ }
+ return Count;
+}
+
+static FORCEINLINE VOID
+TargetCancelRequestList(
+ IN PXENVBD_TARGET Target,
+ IN PLIST_ENTRY List
+ )
+{
+ for (;;) {
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY Entry;
+
+ Entry = RemoveHeadList(List);
+ if (Entry == List)
+ break;
+
+ Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+ TargetPutRequest(Target, Request);
+ }
+}
+
+__checkReturn
+static BOOLEAN
+PrepareReadWrite(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
+ ULONG64 SectorStart = Cdb_LogicalBlock(Srb);
+ ULONG SectorsLeft = Cdb_TransferBlock(Srb);
+ LIST_ENTRY List;
+ XENVBD_SG_LIST SGList;
+ ULONG DebugCount;
+
+ Srb->SrbStatus = SRB_STATUS_PENDING;
+
+ InitializeListHead(&List);
+ SrbExt->Count = 0;
+
+ RtlZeroMemory(&SGList, sizeof(SGList));
+ SGList.SGList = StorPortGetScatterGatherList(TargetGetAdapter(Target), Srb);
+
+ while (SectorsLeft > 0) {
+ ULONG MaxSegments;
+ ULONG SectorsDone = 0;
+ PXENVBD_REQUEST Request;
+
+ Request = TargetGetRequest(Target);
+ if (Request == NULL)
+ goto fail1;
+ InsertTailList(&List, &Request->Entry);
+ InterlockedIncrement(&SrbExt->Count);
+
+ Request->Srb = Srb;
+ MaxSegments = UseIndirect(Target, SectorsLeft);
+
+ if (!PrepareBlkifReadWrite(Target,
+ Request,
+ &SGList,
+ MaxSegments,
+ SectorStart,
+ SectorsLeft,
+ &SectorsDone))
+ goto fail2;
+
+ if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) {
+ if (!PrepareBlkifIndirect(Target, Request))
+ goto fail3;
+ }
+
+ SectorsLeft -= SectorsDone;
+ SectorStart += SectorsDone;
+ }
+
+ DebugCount = TargetQueueRequestList(Target, &List);
+ if (DebugCount != (ULONG)SrbExt->Count) {
+ Trace("[%u] %d != %u\n", TargetGetTargetId(Target), SrbExt->Count, DebugCount);
+ }
+ return TRUE;
+
+fail3:
+fail2:
+fail1:
+ TargetCancelRequestList(Target, &List);
+ SrbExt->Count = 0;
+ Srb->SrbStatus = SRB_STATUS_ERROR;
+ return FALSE;
+}
+
+__checkReturn
+static BOOLEAN
+PrepareSyncCache(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
+ PXENVBD_REQUEST Request;
+ LIST_ENTRY List;
+ UCHAR Operation;
+ ULONG DebugCount;
+
+ Srb->SrbStatus = SRB_STATUS_PENDING;
+
+ if (FrontendGetDiskInfo(Target->Frontend)->FlushCache)
+ Operation = BLKIF_OP_FLUSH_DISKCACHE;
+ else
+ Operation = BLKIF_OP_WRITE_BARRIER;
+
+ InitializeListHead(&List);
+ SrbExt->Count = 0;
+
+ Request = TargetGetRequest(Target);
+ if (Request == NULL)
+ goto fail1;
+ InsertTailList(&List, &Request->Entry);
+ InterlockedIncrement(&SrbExt->Count);
+
+ Request->Srb = Srb;
+ Request->Operation = Operation;
+ Request->FirstSector = Cdb_LogicalBlock(Srb);
+
+ DebugCount = TargetQueueRequestList(Target, &List);
+ if (DebugCount != (ULONG)SrbExt->Count) {
+ Trace("[%u] %d != %u\n", TargetGetTargetId(Target), SrbExt->Count, DebugCount);
+ }
+ return TRUE;
+
+fail1:
+ TargetCancelRequestList(Target, &List);
+ SrbExt->Count = 0;
+ Srb->SrbStatus = SRB_STATUS_ERROR;
+ return FALSE;
+}
+
+__checkReturn
+static BOOLEAN
+PrepareUnmap(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
+ PUNMAP_LIST_HEADER Unmap = Srb->DataBuffer;
+ ULONG Count = _byteswap_ushort(*(PUSHORT)Unmap->BlockDescrDataLength) / sizeof(UNMAP_BLOCK_DESCRIPTOR);
+ ULONG Index;
+ LIST_ENTRY List;
+ ULONG DebugCount;
+
+ Srb->SrbStatus = SRB_STATUS_PENDING;
+
+ InitializeListHead(&List);
+ SrbExt->Count = 0;
+
+ for (Index = 0; Index < Count; ++Index) {
+ PUNMAP_BLOCK_DESCRIPTOR Descr = &Unmap->Descriptors[Index];
+ PXENVBD_REQUEST Request;
+
+ Request = TargetGetRequest(Target);
+ if (Request == NULL)
+ goto fail1;
+ InsertTailList(&List, &Request->Entry);
+ InterlockedIncrement(&SrbExt->Count);
+
+ Request->Srb = Srb;
+ Request->Operation = BLKIF_OP_DISCARD;
+ Request->FirstSector = _byteswap_uint64(*(PULONG64)Descr->StartingLba);
+ Request->NrSectors = _byteswap_ulong(*(PULONG)Descr->LbaCount);
+ Request->Flags = 0;
+ }
+
+ DebugCount = TargetQueueRequestList(Target, &List);
+ if (DebugCount != (ULONG)SrbExt->Count) {
+ Trace("[%u] %d != %u\n", TargetGetTargetId(Target), SrbExt->Count, DebugCount);
+ }
+ return TRUE;
+
+fail1:
+ TargetCancelRequestList(Target, &List);
+ SrbExt->Count = 0;
+ Srb->SrbStatus = SRB_STATUS_ERROR;
+ return FALSE;
+}
+
+//=============================================================================
+// Queue-Related
+static FORCEINLINE VOID
+__TargetPauseDataPath(
+ __in PXENVBD_TARGET Target,
+ __in BOOLEAN Timeout
+ )
+{
+ KIRQL Irql;
+ ULONG Requests;
+ ULONG Count = 0;
+ PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Target->Frontend);
+ PXENVBD_BLOCKRING BlockRing = FrontendGetBlockRing(Target->Frontend);
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ ++Target->Paused;
+ KeReleaseSpinLock(&Target->Lock, Irql);
+
+ Requests = QueueCount(&Target->SubmittedReqs);
+ KeMemoryBarrier();
+
+ Verbose("Target[%d] : Waiting for %d Submitted requests\n", TargetGetTargetId(Target), Requests);
+
+ // poll ring and send event channel notification every 1ms (for up to 3 minutes)
+ while (QueueCount(&Target->SubmittedReqs)) {
+ if (Timeout && Count > 180000)
+ break;
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ BlockRingPoll(BlockRing);
+ KeLowerIrql(Irql);
+ NotifierSend(Notifier); // let backend know it needs to do some work
+ StorPortStallExecution(1000); // 1000 micro-seconds
+ ++Count;
+ }
+
+ Verbose("Target[%d] : %u/%u Submitted requests left (%u iterrations)\n",
+ TargetGetTargetId(Target), QueueCount(&Target->SubmittedReqs), Requests, Count);
+
+ // Abort Fresh SRBs
+ for (;;) {
+ PXENVBD_SRBEXT SrbExt;
+ PLIST_ENTRY Entry = QueuePop(&Target->FreshSrbs);
+ if (Entry == NULL)
+ break;
+ SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
+
+ Verbose("Target[%d] : FreshSrb 0x%p -> SCSI_ABORTED\n", TargetGetTargetId(Target), SrbExt->Srb);
+ SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
+ SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED;
+ AdapterCompleteSrb(TargetGetAdapter(Target), SrbExt->Srb);
+ }
+
+ // Fail PreparedReqs
+ for (;;) {
+ PXENVBD_SRBEXT SrbExt;
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY Entry = QueuePop(&Target->PreparedReqs);
+ if (Entry == NULL)
+ break;
+ Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+ SrbExt = GetSrbExt(Request->Srb);
+
+ Verbose("Target[%d] : PreparedReq 0x%p -> FAILED\n", TargetGetTargetId(Target), Request);
+
+ SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
+ TargetPutRequest(Target, Request);
+
+ if (InterlockedDecrement(&SrbExt->Count) == 0) {
+ SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+ AdapterCompleteSrb(TargetGetAdapter(Target), SrbExt->Srb);
+ }
+ }
+}
+
+static FORCEINLINE VOID
+__TargetUnpauseDataPath(
+ __in PXENVBD_TARGET Target
+ )
+{
+ KIRQL Irql;
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ --Target->Paused;
+ KeReleaseSpinLock(&Target->Lock, Irql);
+}
+
+static FORCEINLINE BOOLEAN
+TargetPrepareFresh(
+ IN PXENVBD_TARGET Target
+ )
+{
+ PXENVBD_SRBEXT SrbExt;
+ PLIST_ENTRY Entry;
+
+ Entry = QueuePop(&Target->FreshSrbs);
+ if (Entry == NULL)
+ return FALSE; // fresh queue is empty
+
+ SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
+
+ switch (Cdb_OperationEx(SrbExt->Srb)) {
+ case SCSIOP_READ:
+ case SCSIOP_WRITE:
+ if (PrepareReadWrite(Target, SrbExt->Srb))
+ return TRUE; // prepared this SRB
+ break;
+ case SCSIOP_SYNCHRONIZE_CACHE:
+ if (PrepareSyncCache(Target, SrbExt->Srb))
+ return TRUE; // prepared this SRB
+ break;
+ case SCSIOP_UNMAP:
+ if (PrepareUnmap(Target, SrbExt->Srb))
+ return TRUE; // prepared this SRB
+ break;
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+ QueueUnPop(&Target->FreshSrbs, &SrbExt->Entry);
+
+ return FALSE; // prepare failed
+}
+
+static FORCEINLINE BOOLEAN
+TargetSubmitPrepared(
+ __in PXENVBD_TARGET Target
+ )
+{
+ PXENVBD_BLOCKRING BlockRing = FrontendGetBlockRing(Target->Frontend);
+ if (TargetIsPaused(Target)) {
+ if (QueueCount(&Target->PreparedReqs))
+ Warning("Target[%d] : Paused, not submitting new requests (%u)\n",
+ TargetGetTargetId(Target),
+ QueueCount(&Target->PreparedReqs));
+ return FALSE;
+ }
+
+ for (;;) {
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY Entry;
+
+ Entry = QueuePop(&Target->PreparedReqs);
+ if (Entry == NULL)
+ break;
+
+ Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+
+ QueueAppend(&Target->SubmittedReqs, &Request->Entry);
+ KeMemoryBarrier();
+
+ if (BlockRingSubmit(BlockRing, Request))
+ continue;
+
+ QueueRemove(&Target->SubmittedReqs, &Request->Entry);
+ QueueUnPop(&Target->PreparedReqs, &Request->Entry);
+ return FALSE; // ring full
+ }
+
+ return TRUE;
+}
+
+static FORCEINLINE VOID
+TargetCompleteShutdown(
+ __in PXENVBD_TARGET Target
+ )
+{
+ if (QueueCount(&Target->ShutdownSrbs) == 0)
+ return;
+
+ if (QueueCount(&Target->FreshSrbs) ||
+ QueueCount(&Target->PreparedReqs) ||
+ QueueCount(&Target->SubmittedReqs))
+ return;
+
+ for (;;) {
+ PXENVBD_SRBEXT SrbExt;
+ PLIST_ENTRY Entry = QueuePop(&Target->ShutdownSrbs);
+ if (Entry == NULL)
+ break;
+ SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
+ SrbExt->Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ AdapterCompleteSrb(TargetGetAdapter(Target), SrbExt->Srb);
+ }
+}
+
+static FORCEINLINE PCHAR
+BlkifOperationName(
+ IN UCHAR Operation
+ )
+{
+ switch (Operation) {
+ case BLKIF_OP_READ: return "READ";
+ case BLKIF_OP_WRITE: return "WRITE";
+ case BLKIF_OP_WRITE_BARRIER: return "WRITE_BARRIER";
+ case BLKIF_OP_FLUSH_DISKCACHE: return "FLUSH_DISKCACHE";
+ case BLKIF_OP_RESERVED_1: return "RESERVED_1";
+ case BLKIF_OP_DISCARD: return "DISCARD";
+ case BLKIF_OP_INDIRECT: return "INDIRECT";
+ default: return "<unknown>";
+ }
+}
+
+VOID
+TargetSubmitRequests(
+ __in PXENVBD_TARGET Target
+ )
+{
+ for (;;) {
+ // submit all prepared requests (0 or more requests)
+ // return TRUE if submitted 0 or more requests from prepared queue
+ // return FALSE iff ring is full
+ if (!TargetSubmitPrepared(Target))
+ break;
+
+ // prepare a single SRB (into 1 or more requests)
+ // return TRUE if prepare succeeded
+ // return FALSE if prepare failed or fresh queue empty
+ if (!TargetPrepareFresh(Target))
+ break;
+ }
+
+ // if no requests/SRBs outstanding, complete any shutdown SRBs
+ TargetCompleteShutdown(Target);
+}
+
+VOID
+TargetCompleteResponse(
+ __in PXENVBD_TARGET Target,
+ __in ULONG Tag,
+ __in SHORT Status
+ )
+{
+ PXENVBD_REQUEST Request;
+ PSCSI_REQUEST_BLOCK Srb;
+ PXENVBD_SRBEXT SrbExt;
+
+ Request = TargetRequestFromTag(Target, Tag);
+ if (Request == NULL)
+ return;
+
+ Srb = Request->Srb;
+ SrbExt = GetSrbExt(Srb);
+ ASSERT3P(SrbExt, !=, NULL);
+
+ switch (Status) {
+ case BLKIF_RSP_OKAY:
+ RequestCopyOutput(Request);
+ break;
+
+ case BLKIF_RSP_EOPNOTSUPP:
+ // Remove appropriate feature support
+ FrontendRemoveFeature(Target->Frontend, Request->Operation);
+ // Succeed this SRB, subsiquent SRBs will be succeeded instead of being passed to the backend.
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ break;
+
+ case BLKIF_RSP_ERROR:
+ default:
+ Warning("Target[%d] : %s BLKIF_RSP_ERROR (Tag %x)\n",
+ TargetGetTargetId(Target), BlkifOperationName(Request->Operation), Tag);
+ Srb->SrbStatus = SRB_STATUS_ERROR;
+ break;
+ }
+
+ TargetPutRequest(Target, Request);
+
+ // complete srb
+ if (InterlockedDecrement(&SrbExt->Count) == 0) {
+ if (Srb->SrbStatus == SRB_STATUS_PENDING) {
+ // SRB has not hit a failure condition (BLKIF_RSP_ERROR | BLKIF_RSP_EOPNOTSUPP)
+ // from any of its responses. SRB must have succeeded
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ Srb->ScsiStatus = 0x00; // SCSI_GOOD
+ } else {
+ // Srb->SrbStatus has already been set by 1 or more requests with Status != BLKIF_RSP_OKAY
+ Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+ }
+
+ AdapterCompleteSrb(TargetGetAdapter(Target), Srb);
+ }
+}
+
+VOID
+TargetPreResume(
+ __in PXENVBD_TARGET Target
+ )
+{
+ LIST_ENTRY List;
+
+ InitializeListHead(&List);
+
+ // pop all submitted requests, cleanup and add associated SRB to a list
+ for (;;) {
+ PXENVBD_SRBEXT SrbExt;
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY Entry = QueuePop(&Target->SubmittedReqs);
+ if (Entry == NULL)
+ break;
+ Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+ SrbExt = GetSrbExt(Request->Srb);
+
+ TargetPutRequest(Target, Request);
+
+ if (InterlockedDecrement(&SrbExt->Count) == 0) {
+ InsertTailList(&List, &SrbExt->Entry);
+ }
+ }
+
+ // pop all prepared requests, cleanup and add associated SRB to a list
+ for (;;) {
+ PXENVBD_SRBEXT SrbExt;
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY Entry = QueuePop(&Target->PreparedReqs);
+ if (Entry == NULL)
+ break;
+ Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+ SrbExt = GetSrbExt(Request->Srb);
+
+ TargetPutRequest(Target, Request);
+
+ if (InterlockedDecrement(&SrbExt->Count) == 0) {
+ InsertTailList(&List, &SrbExt->Entry);
+ }
+ }
+
+ // foreach SRB in list, put on start of FreshSrbs
+ for (;;) {
+ PXENVBD_SRBEXT SrbExt;
+ PLIST_ENTRY Entry = RemoveTailList(&List);
+ if (Entry == &List)
+ break;
+ SrbExt = CONTAINING_RECORD(Entry, XENVBD_SRBEXT, Entry);
+
+ QueueUnPop(&Target->FreshSrbs, &SrbExt->Entry);
+ }
+
+ // now the first set of requests popped off submitted list is the next SRB
+ // to be popped off the fresh list
+}
+
+VOID
+TargetPostResume(
+ __in PXENVBD_TARGET Target
+ )
+{
+ KIRQL Irql;
+
+ Verbose("Target[%d] : %d Fresh SRBs\n", TargetGetTargetId(Target), QueueCount(&Target->FreshSrbs));
+
+ // clear missing flag
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ Verbose("Target[%d] : %s (%s)\n", TargetGetTargetId(Target), Target->Missing ? "MISSING" : "NOT_MISSING", Target->Reason);
+ Target->Missing = FALSE;
+ Target->Reason = NULL;
+ KeReleaseSpinLock(&Target->Lock, Irql);
+}
+
+//=============================================================================
+// SRBs
+__checkReturn
+static FORCEINLINE BOOLEAN
+__ValidateSectors(
+ __in ULONG64 SectorCount,
+ __in ULONG64 Start,
+ __in ULONG Length
+ )
+{
+ // Deal with overflow
+ return (Start < SectorCount) && ((Start + Length) <= SectorCount);
+}
+
+__checkReturn
+static FORCEINLINE BOOLEAN
+__ValidateSrbBuffer(
+ __in PCHAR Caller,
+ __in PSCSI_REQUEST_BLOCK Srb,
+ __in ULONG MinLength
+ )
+{
+ if (Srb->DataBuffer == NULL) {
+ Error("%s: Srb[0x%p].DataBuffer = NULL\n", Caller, Srb);
+ return FALSE;
+ }
+ if (MinLength) {
+ if (Srb->DataTransferLength < MinLength) {
+ Error("%s: Srb[0x%p].DataTransferLength < %d\n", Caller, Srb, MinLength);
+ return FALSE;
+ }
+ } else {
+ if (Srb->DataTransferLength == 0) {
+ Error("%s: Srb[0x%p].DataTransferLength = 0\n", Caller, Srb);
+ return FALSE;
+ }
+ }
+
+ return TRUE;
+}
+
+__checkReturn
+static DECLSPEC_NOINLINE BOOLEAN
+TargetReadWrite(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PXENVBD_DISKINFO DiskInfo = FrontendGetDiskInfo(Target->Frontend);
+ PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
+ PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Target->Frontend);
+
+ if (FrontendGetCaps(Target->Frontend)->Connected == FALSE) {
+ Trace("Target[%d] : Not Ready, fail SRB\n", TargetGetTargetId(Target));
+ Srb->ScsiStatus = 0x40; // SCSI_ABORT;
+ return TRUE;
+ }
+
+ // check valid sectors
+ if (!__ValidateSectors(DiskInfo->SectorCount, Cdb_LogicalBlock(Srb), Cdb_TransferBlock(Srb))) {
+ Trace("Target[%d] : Invalid Sector (%d @ %lld < %lld)\n", TargetGetTargetId(Target), Cdb_TransferBlock(Srb), Cdb_LogicalBlock(Srb), DiskInfo->SectorCount);
+ Srb->ScsiStatus = 0x40; // SCSI_ABORT
+ return TRUE; // Complete now
+ }
+
+ QueueAppend(&Target->FreshSrbs, &SrbExt->Entry);
+ NotifierKick(Notifier);
+
+ return FALSE;
+}
+
+__checkReturn
+static DECLSPEC_NOINLINE BOOLEAN
+TargetSyncCache(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
+ PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Target->Frontend);
+
+ if (FrontendGetCaps(Target->Frontend)->Connected == FALSE) {
+ Trace("Target[%d] : Not Ready, fail SRB\n", TargetGetTargetId(Target));
+ Srb->ScsiStatus = 0x40; // SCSI_ABORT;
+ return TRUE;
+ }
+
+ if (FrontendGetDiskInfo(Target->Frontend)->FlushCache == FALSE &&
+ FrontendGetDiskInfo(Target->Frontend)->Barrier == FALSE) {
+ Trace("Target[%d] : FLUSH and BARRIER not supported, suppressing\n", TargetGetTargetId(Target));
+ Srb->ScsiStatus = 0x00; // SCSI_GOOD
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ return TRUE;
+ }
+
+ QueueAppend(&Target->FreshSrbs, &SrbExt->Entry);
+ NotifierKick(Notifier);
+
+ return FALSE;
+}
+
+__checkReturn
+static DECLSPEC_NOINLINE BOOLEAN
+TargetUnmap(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
+ PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Target->Frontend);
+
+ if (FrontendGetCaps(Target->Frontend)->Connected == FALSE) {
+ Trace("Target[%d] : Not Ready, fail SRB\n", TargetGetTargetId(Target));
+ Srb->ScsiStatus = 0x40; // SCSI_ABORT;
+ return TRUE;
+ }
+
+ if (FrontendGetDiskInfo(Target->Frontend)->Discard == FALSE) {
+ Trace("Target[%d] : DISCARD not supported, suppressing\n", TargetGetTargetId(Target));
+ Srb->ScsiStatus = 0x00; // SCSI_GOOD
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ return TRUE;
+ }
+
+ QueueAppend(&Target->FreshSrbs, &SrbExt->Entry);
+ NotifierKick(Notifier);
+
+ return FALSE;
+}
+
+#define MODE_CACHING_PAGE_LENGTH 20
+static DECLSPEC_NOINLINE VOID
+TargetModeSense(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PMODE_PARAMETER_HEADER Header = Srb->DataBuffer;
+ const UCHAR PageCode = Cdb_PageCode(Srb);
+ ULONG LengthLeft = Cdb_AllocationLength(Srb);
+ PVOID CurrentPage = Srb->DataBuffer;
+
+ UNREFERENCED_PARAMETER(Target);
+
+ RtlZeroMemory(Srb->DataBuffer, Srb->DataTransferLength);
+
+ if (!__ValidateSrbBuffer(__FUNCTION__, Srb, (ULONG)sizeof(struct _MODE_SENSE))) {
+ Srb->ScsiStatus = 0x40;
+ Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
+ Srb->DataTransferLength = 0;
+ return;
+ }
+
+ // TODO : CDROM requires more ModePage entries
+ // Header
+ Header->ModeDataLength = sizeof(MODE_PARAMETER_HEADER) - 1;
+ Header->MediumType = 0;
+ Header->DeviceSpecificParameter = 0;
+ Header->BlockDescriptorLength = 0;
+ LengthLeft -= sizeof(MODE_PARAMETER_HEADER);
+ CurrentPage = ((PUCHAR)CurrentPage + sizeof(MODE_PARAMETER_HEADER));
+
+ // Fill in Block Parameters (if Specified and space)
+ // when the DBD (Disable Block Descriptor) is set, ignore the block page
+ if (Cdb_Dbd(Srb) == 0 &&
+ LengthLeft >= sizeof(MODE_PARAMETER_BLOCK)) {
+ PMODE_PARAMETER_BLOCK Block = (PMODE_PARAMETER_BLOCK)CurrentPage;
+ // Fill in BlockParams
+ Block->DensityCode = 0;
+ Block->NumberOfBlocks[0] = 0;
+ Block->NumberOfBlocks[1] = 0;
+ Block->NumberOfBlocks[2] = 0;
+ Block->BlockLength[0] = 0;
+ Block->BlockLength[1] = 0;
+ Block->BlockLength[2] = 0;
+
+ Header->BlockDescriptorLength = sizeof(MODE_PARAMETER_BLOCK);
+ Header->ModeDataLength += sizeof(MODE_PARAMETER_BLOCK);
+ LengthLeft -= sizeof(MODE_PARAMETER_BLOCK);
+ CurrentPage = ((PUCHAR)CurrentPage + sizeof(MODE_PARAMETER_BLOCK));
+ }
+
+ // Fill in Cache Parameters (if Specified and space)
+ if ((PageCode == MODE_PAGE_CACHING || PageCode == MODE_SENSE_RETURN_ALL) &&
+ LengthLeft >= MODE_CACHING_PAGE_LENGTH) {
+ PMODE_CACHING_PAGE Caching = (PMODE_CACHING_PAGE)CurrentPage;
+ // Fill in CachingParams
+ Caching->PageCode = MODE_PAGE_CACHING;
+ Caching->PageSavable = 0;
+ Caching->PageLength = MODE_CACHING_PAGE_LENGTH;
+ Caching->ReadDisableCache = 0;
+ Caching->MultiplicationFactor = 0;
+ Caching->WriteCacheEnable = FrontendGetDiskInfo(Target->Frontend)->FlushCache ? 1 : 0;
+ Caching->WriteRetensionPriority = 0;
+ Caching->ReadRetensionPriority = 0;
+ Caching->DisablePrefetchTransfer[0] = 0;
+ Caching->DisablePrefetchTransfer[1] = 0;
+ Caching->MinimumPrefetch[0] = 0;
+ Caching->MinimumPrefetch[1] = 0;
+ Caching->MaximumPrefetch[0] = 0;
+ Caching->MaximumPrefetch[1] = 0;
+ Caching->MaximumPrefetchCeiling[0] = 0;
+ Caching->MaximumPrefetchCeiling[1] = 0;
+
+ Header->ModeDataLength += MODE_CACHING_PAGE_LENGTH;
+ LengthLeft -= MODE_CACHING_PAGE_LENGTH;
+ CurrentPage = ((PUCHAR)CurrentPage + MODE_CACHING_PAGE_LENGTH);
+ }
+
+ // Fill in Informational Exception Parameters (if Specified and space)
+ if ((PageCode == MODE_PAGE_FAULT_REPORTING || PageCode == MODE_SENSE_RETURN_ALL) &&
+ LengthLeft >= sizeof(MODE_INFO_EXCEPTIONS)) {
+ PMODE_INFO_EXCEPTIONS Exceptions = (PMODE_INFO_EXCEPTIONS)CurrentPage;
+ // Fill in Exceptions
+ Exceptions->PageCode = MODE_PAGE_FAULT_REPORTING;
+ Exceptions->PSBit = 0;
+ Exceptions->PageLength = sizeof(MODE_INFO_EXCEPTIONS);
+ Exceptions->Flags = 0;
+ Exceptions->Dexcpt = 1; // disabled
+ Exceptions->ReportMethod = 0;
+ Exceptions->IntervalTimer[0] = 0;
+ Exceptions->IntervalTimer[1] = 0;
+ Exceptions->IntervalTimer[2] = 0;
+ Exceptions->IntervalTimer[3] = 0;
+ Exceptions->ReportCount[0] = 0;
+ Exceptions->ReportCount[1] = 0;
+ Exceptions->ReportCount[2] = 0;
+ Exceptions->ReportCount[3] = 0;
+
+ Header->ModeDataLength += sizeof(MODE_INFO_EXCEPTIONS);
+ LengthLeft -= sizeof(MODE_INFO_EXCEPTIONS);
+ CurrentPage = ((PUCHAR)CurrentPage + sizeof(MODE_INFO_EXCEPTIONS));
+ }
+
+ // Finish this SRB
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ Srb->DataTransferLength = __min(Cdb_AllocationLength(Srb), (ULONG)(Header->ModeDataLength + 1));
+}
+
+static DECLSPEC_NOINLINE VOID
+TargetRequestSense(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PSENSE_DATA Sense = Srb->DataBuffer;
+
+ UNREFERENCED_PARAMETER(Target);
+
+ if (!__ValidateSrbBuffer(__FUNCTION__, Srb, (ULONG)sizeof(SENSE_DATA))) {
+ Srb->ScsiStatus = 0x40;
+ Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
+ return;
+ }
+
+ RtlZeroMemory(Sense, sizeof(SENSE_DATA));
+
+ Sense->ErrorCode = 0x70;
+ Sense->Valid = 1;
+ Sense->AdditionalSenseCodeQualifier = 0;
+ Sense->SenseKey = SCSI_SENSE_NO_SENSE;
+ Sense->AdditionalSenseCode = SCSI_ADSENSE_NO_SENSE;
+ Srb->DataTransferLength = sizeof(SENSE_DATA);
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+}
+
+static DECLSPEC_NOINLINE VOID
+TargetReportLuns(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ ULONG Length;
+ ULONG Offset;
+ ULONG AllocLength = Cdb_AllocationLength(Srb);
+ PUCHAR Buffer = Srb->DataBuffer;
+
+ UNREFERENCED_PARAMETER(Target);
+
+ if (!__ValidateSrbBuffer(__FUNCTION__, Srb, 8)) {
+ Srb->ScsiStatus = 0x40;
+ Srb->SrbStatus = SRB_STATUS_DATA_OVERRUN;
+ Srb->DataTransferLength = 0;
+ return;
+ }
+
+ RtlZeroMemory(Buffer, AllocLength);
+
+ Length = 0;
+ Offset = 8;
+
+ if (Offset + 8 <= AllocLength) {
+ Buffer[Offset] = 0;
+ Offset += 8;
+ Length += 8;
+ }
+
+ if (Offset + 8 <= AllocLength) {
+ Buffer[Offset] = XENVBD_MAX_TARGETS;
+ Offset += 8;
+ Length += 8;
+ }
+
+ REVERSE_BYTES(Buffer, &Length);
+
+ Srb->DataTransferLength = __min(Length, AllocLength);
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+}
+
+static DECLSPEC_NOINLINE VOID
+TargetReadCapacity(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PREAD_CAPACITY_DATA Capacity = Srb->DataBuffer;
+ PXENVBD_DISKINFO DiskInfo = FrontendGetDiskInfo(Target->Frontend);
+ ULONG64 SectorCount;
+ ULONG SectorSize;
+ ULONG LastBlock;
+
+ if (Cdb_PMI(Srb) == 0 && Cdb_LogicalBlock(Srb) != 0) {
+ Srb->ScsiStatus = 0x02; // CHECK_CONDITION
+ return;
+ }
+
+ SectorCount = DiskInfo->SectorCount;
+ SectorSize = DiskInfo->SectorSize;
+
+ if (SectorCount == (ULONG)SectorCount)
+ LastBlock = (ULONG)SectorCount - 1;
+ else
+ LastBlock = ~(ULONG)0;
+
+ if (Capacity) {
+ Capacity->LogicalBlockAddress = _byteswap_ulong(LastBlock);
+ Capacity->BytesPerBlock = _byteswap_ulong(SectorSize);
+ }
+
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+}
+
+static DECLSPEC_NOINLINE VOID
+TargetReadCapacity16(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PREAD_CAPACITY16_DATA Capacity = Srb->DataBuffer;
+ PXENVBD_DISKINFO DiskInfo = FrontendGetDiskInfo(Target->Frontend);
+ ULONG64 SectorCount;
+ ULONG SectorSize;
+ ULONG PhysSectorSize;
+ ULONG LogicalPerPhysical;
+ ULONG LogicalPerPhysicalExponent;
+
+ if (Cdb_PMI(Srb) == 0 && Cdb_LogicalBlock(Srb) != 0) {
+ Srb->ScsiStatus = 0x02; // CHECK_CONDITION
+ return;
+ }
+
+ SectorCount = DiskInfo->SectorCount;
+ SectorSize = DiskInfo->SectorSize;
+ PhysSectorSize = DiskInfo->PhysSectorSize;
+
+ LogicalPerPhysical = PhysSectorSize / SectorSize;
+
+ if (!_BitScanReverse(&LogicalPerPhysicalExponent, LogicalPerPhysical))
+ LogicalPerPhysicalExponent = 0;
+
+ if (Capacity) {
+ Capacity->LogicalBlockAddress.QuadPart = _byteswap_uint64(SectorCount - 1);
+ Capacity->BytesPerBlock = _byteswap_ulong(SectorSize);
+ Capacity->LogicalPerPhysicalExponent = (UCHAR)LogicalPerPhysicalExponent;
+ }
+
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+}
+
+//=============================================================================
+// StorPort Methods
+__checkReturn
+static FORCEINLINE BOOLEAN
+__TargetExecuteScsi(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ const UCHAR Operation = Cdb_OperationEx(Srb);
+ PXENVBD_DISKINFO DiskInfo = FrontendGetDiskInfo(Target->Frontend);
+
+ if (DiskInfo->DiskInfo & VDISK_READONLY) {
+ Trace("Target[%d] : (%08x) Read-Only, fail SRB (%02x:%s)\n", TargetGetTargetId(Target),
+ DiskInfo->DiskInfo, Operation, Cdb_OperationName(Operation));
+ Srb->ScsiStatus = 0x40; // SCSI_ABORT
+ return TRUE;
+ }
+
+ // idea: check pdo state here. still push to freshsrbs
+ switch (Operation) {
+ case SCSIOP_READ:
+ case SCSIOP_WRITE:
+ return TargetReadWrite(Target, Srb);
+ break;
+
+ case SCSIOP_SYNCHRONIZE_CACHE:
+ return TargetSyncCache(Target, Srb);
+ break;
+
+ case SCSIOP_UNMAP:
+ return TargetUnmap(Target, Srb);
+ break;
+
+ case SCSIOP_INQUIRY:
+ if (!StorPortSetDeviceQueueDepth(TargetGetAdapter(Target),
+ 0,
+ (UCHAR)TargetGetTargetId(Target),
+ 0,
+ XENVBD_MAX_QUEUE_DEPTH))
+ Verbose("Target[%d] : Failed to set queue depth\n",
+ TargetGetTargetId(Target));
+ PdoInquiry(TargetGetTargetId(Target), FrontendGetInquiry(Target->Frontend), Srb, Target->DeviceType);
+ break;
+ case SCSIOP_MODE_SENSE:
+ TargetModeSense(Target, Srb);
+ break;
+ case SCSIOP_REQUEST_SENSE:
+ TargetRequestSense(Target, Srb);
+ break;
+ case SCSIOP_REPORT_LUNS:
+ TargetReportLuns(Target, Srb);
+ break;
+ case SCSIOP_READ_CAPACITY:
+ TargetReadCapacity(Target, Srb);
+ break;
+ case SCSIOP_READ_CAPACITY16:
+ TargetReadCapacity16(Target, Srb);
+ break;
+ case SCSIOP_MEDIUM_REMOVAL:
+ case SCSIOP_TEST_UNIT_READY:
+ case SCSIOP_RESERVE_UNIT:
+ case SCSIOP_RESERVE_UNIT10:
+ case SCSIOP_RELEASE_UNIT:
+ case SCSIOP_RELEASE_UNIT10:
+ case SCSIOP_VERIFY:
+ case SCSIOP_VERIFY16:
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ break;
+ case SCSIOP_START_STOP_UNIT:
+ Trace("Target[%d] : Start/Stop Unit (%02X)\n", TargetGetTargetId(Target), Srb->Cdb[4]);
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+ break;
+ default:
+ Trace("Target[%d] : Unsupported CDB (%02x:%s)\n", TargetGetTargetId(Target), Operation, Cdb_OperationName(Operation));
+ break;
+ }
+ return TRUE;
+}
+
+static FORCEINLINE BOOLEAN
+__TargetQueueShutdown(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ PXENVBD_SRBEXT SrbExt = GetSrbExt(Srb);
+ PXENVBD_NOTIFIER Notifier = FrontendGetNotifier(Target->Frontend);
+
+ QueueAppend(&Target->ShutdownSrbs, &SrbExt->Entry);
+ NotifierKick(Notifier);
+
+ return FALSE;
+}
+
+static FORCEINLINE BOOLEAN
+__TargetReset(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ Verbose("Target[%u] ====>\n", TargetGetTargetId(Target));
+
+ TargetReset(Target);
+ Srb->SrbStatus = SRB_STATUS_SUCCESS;
+
+ Verbose("Target[%u] <====\n", TargetGetTargetId(Target));
+ return TRUE;
+}
+
+__checkReturn
+static FORCEINLINE BOOLEAN
+__ValidateSrbForTarget(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ const UCHAR Operation = Cdb_OperationEx(Srb);
+
+ if (Target == NULL) {
+ Error("Invalid Target(NULL) (%02x:%s)\n",
+ Operation, Cdb_OperationName(Operation));
+ Srb->SrbStatus = SRB_STATUS_INVALID_TARGET_ID;
+ return FALSE;
+ }
+
+ if (Srb->PathId != 0) {
+ Error("Target[%d] : Invalid PathId(%d) (%02x:%s)\n",
+ TargetGetTargetId(Target), Srb->PathId, Operation, Cdb_OperationName(Operation));
+ Srb->SrbStatus = SRB_STATUS_INVALID_PATH_ID;
+ return FALSE;
+ }
+
+ if (Srb->Lun != 0) {
+ Error("Target[%d] : Invalid Lun(%d) (%02x:%s)\n",
+ TargetGetTargetId(Target), Srb->Lun, Operation, Cdb_OperationName(Operation));
+ Srb->SrbStatus = SRB_STATUS_INVALID_LUN;
+ return FALSE;
+ }
+
+ if (TargetIsMissing(Target)) {
+ Error("Target[%d] : %s (%s) (%02x:%s)\n",
+ TargetGetTargetId(Target), Target->Missing ? "MISSING" : "NOT_MISSING", Target->Reason, Operation, Cdb_OperationName(Operation));
+ Srb->SrbStatus = SRB_STATUS_NO_DEVICE;
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+__checkReturn
+BOOLEAN
+TargetStartIo(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ )
+{
+ if (!__ValidateSrbForTarget(Target, Srb))
+ return TRUE;
+
+ switch (Srb->Function) {
+ case SRB_FUNCTION_EXECUTE_SCSI:
+ return __TargetExecuteScsi(Target, Srb);
+
+ case SRB_FUNCTION_RESET_DEVICE:
+ return __TargetReset(Target, Srb);
+
+ case SRB_FUNCTION_FLUSH:
+ case SRB_FUNCTION_SHUTDOWN:
+ return __TargetQueueShutdown(Target, Srb);
+
+ default:
+ return TRUE;
+ }
+}
+
+static FORCEINLINE VOID
+__TargetCleanupSubmittedReqs(
+ IN PXENVBD_TARGET Target
+ )
+{
+ // Fail PreparedReqs
+ for (;;) {
+ PXENVBD_SRBEXT SrbExt;
+ PXENVBD_REQUEST Request;
+ PLIST_ENTRY Entry = QueuePop(&Target->SubmittedReqs);
+ if (Entry == NULL)
+ break;
+ Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry);
+ SrbExt = GetSrbExt(Request->Srb);
+
+ Verbose("Target[%d] : SubmittedReq 0x%p -> FAILED\n", TargetGetTargetId(Target), Request);
+
+ TargetPutRequest(Target, Request);
+
+ if (InterlockedDecrement(&SrbExt->Count) == 0) {
+ SrbExt->Srb->SrbStatus = SRB_STATUS_ABORTED;
+ SrbExt->Srb->ScsiStatus = 0x40; // SCSI_ABORTED
+ AdapterCompleteSrb(TargetGetAdapter(Target), SrbExt->Srb);
+ }
+ }
+}
+
+VOID
+TargetReset(
+ __in PXENVBD_TARGET Target
+ )
+{
+ Trace("Target[%d] ====> (Irql=%d)\n", TargetGetTargetId(Target), KeGetCurrentIrql());
+
+ __TargetPauseDataPath(Target, TRUE);
+
+ if (QueueCount(&Target->SubmittedReqs)) {
+ Error("Target[%d] : backend has %u outstanding requests after a TargetReset\n",
+ TargetGetTargetId(Target), QueueCount(&Target->SubmittedReqs));
+ }
+
+ __TargetUnpauseDataPath(Target);
+
+ Trace("Target[%d] <==== (Irql=%d)\n", TargetGetTargetId(Target), KeGetCurrentIrql());
+}
+
+
+VOID
+TargetSrbPnp(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_PNP_REQUEST_BLOCK Srb
+ )
+{
+ switch (Srb->PnPAction) {
+ case StorQueryCapabilities: {
+ PSTOR_DEVICE_CAPABILITIES DeviceCaps = Srb->DataBuffer;
+ PXENVBD_CAPS Caps = FrontendGetCaps(Target->Frontend);
+
+ if (Caps->Removable)
+ DeviceCaps->Removable = 1;
+ if (Caps->Removable)
+ DeviceCaps->EjectSupported = 1;
+ if (Caps->SurpriseRemovable)
+ DeviceCaps->SurpriseRemovalOK = 1;
+
+ DeviceCaps->UniqueID = 1;
+
+ } break;
+
+ default:
+ break;
+ }
+}
+
+//=============================================================================
+// PnP Handler
+static FORCEINLINE VOID
+__TargetDeviceUsageNotification(
+ __in PXENVBD_TARGET Target,
+ __in PIRP Irp
+ )
+{
+ PIO_STACK_LOCATION StackLocation;
+ BOOLEAN Value;
+ DEVICE_USAGE_NOTIFICATION_TYPE Type;
+ PXENVBD_CAPS Caps = FrontendGetCaps(Target->Frontend);
+
+ StackLocation = IoGetCurrentIrpStackLocation(Irp);
+ Value = StackLocation->Parameters.UsageNotification.InPath;
+ Type = StackLocation->Parameters.UsageNotification.Type;
+
+ switch (Type) {
+ case DeviceUsageTypePaging:
+ if (Caps->Paging == Value)
+ return;
+ Caps->Paging = Value;
+ break;
+
+ case DeviceUsageTypeHibernation:
+ if (Caps->Hibernation == Value)
+ return;
+ Caps->Hibernation = Value;
+ break;
+
+ case DeviceUsageTypeDumpFile:
+ if (Caps->DumpFile == Value)
+ return;
+ Caps->DumpFile = Value;
+ break;
+
+ default:
+ return;
+ }
+ FrontendWriteUsage(Target->Frontend);
+}
+
+static FORCEINLINE VOID
+__TargetCheckEjectPending(
+ __in PXENVBD_TARGET Target
+ )
+{
+ KIRQL Irql;
+ BOOLEAN EjectPending = FALSE;
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ if (Target->EjectPending) {
+ EjectPending = TRUE;
+ Target->EjectPending = FALSE;
+ Target->EjectRequested = TRUE;
+ }
+ KeReleaseSpinLock(&Target->Lock, Irql);
+
+ if (EjectPending) {
+ Verbose("Target[%d] : IoRequestDeviceEject(0x%p)\n", TargetGetTargetId(Target), Target->DeviceObject);
+ IoRequestDeviceEject(Target->DeviceObject);
+ }
+}
+
+static FORCEINLINE VOID
+__TargetCheckEjectFailed(
+ __in PXENVBD_TARGET Target
+ )
+{
+ KIRQL Irql;
+ BOOLEAN EjectFailed = FALSE;
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ if (Target->EjectRequested) {
+ EjectFailed = TRUE;
+ Target->EjectRequested = FALSE;
+ }
+ KeReleaseSpinLock(&Target->Lock, Irql);
+
+ if (EjectFailed) {
+ Error("Target[%d] : Unplug failed due to open handle(s)!\n", TargetGetTargetId(Target));
+ FrontendStoreWriteFrontend(Target->Frontend, "error", "Unplug failed due to open handle(s)!");
+ }
+}
+
+static FORCEINLINE VOID
+__TargetRemoveDevice(
+ __in PXENVBD_TARGET Target
+ )
+{
+ TargetD0ToD3(Target);
+
+ switch (TargetGetDevicePnpState(Target)) {
+ case SurpriseRemovePending:
+ TargetSetMissing(Target, "Surprise Remove");
+ TargetSetDevicePnpState(Target, Deleted);
+ StorPortNotification(BusChangeDetected, TargetGetAdapter(Target), 0);
+ break;
+
+ default:
+ TargetSetMissing(Target, "Removed");
+ TargetSetDevicePnpState(Target, Deleted);
+ StorPortNotification(BusChangeDetected, TargetGetAdapter(Target), 0);
+ break;
+ }
+}
+
+static FORCEINLINE VOID
+__TargetEject(
+ __in PXENVBD_TARGET Target
+ )
+{
+ TargetSetMissing(Target, "Ejected");
+ TargetSetDevicePnpState(Target, Deleted);
+ StorPortNotification(BusChangeDetected, TargetGetAdapter(Target), 0);
+}
+
+__checkReturn
+NTSTATUS
+TargetDispatchPnp(
+ __in PXENVBD_TARGET Target,
+ __in PDEVICE_OBJECT DeviceObject,
+ __in PIRP Irp
+ )
+{
+ PIO_STACK_LOCATION Stack = IoGetCurrentIrpStackLocation(Irp);
+
+ __TargetCheckEjectPending(Target);
+
+ switch (Stack->MinorFunction) {
+ case IRP_MN_START_DEVICE:
+ (VOID) TargetD3ToD0(Target);
+ TargetSetDevicePnpState(Target, Started);
+ break;
+
+ case IRP_MN_QUERY_STOP_DEVICE:
+ TargetSetDevicePnpState(Target, StopPending);
+ break;
+
+ case IRP_MN_CANCEL_STOP_DEVICE:
+ __TargetRestoreDevicePnpState(Target, StopPending);
+ break;
+
+ case IRP_MN_STOP_DEVICE:
+ TargetD0ToD3(Target);
+ TargetSetDevicePnpState(Target, Stopped);
+ break;
+
+ case IRP_MN_QUERY_REMOVE_DEVICE:
+ TargetSetDevicePnpState(Target, RemovePending);
+ break;
+
+ case IRP_MN_CANCEL_REMOVE_DEVICE:
+ __TargetCheckEjectFailed(Target);
+ __TargetRestoreDevicePnpState(Target, RemovePending);
+ break;
+
+ case IRP_MN_SURPRISE_REMOVAL:
+ TargetSetDevicePnpState(Target, SurpriseRemovePending);
+ break;
+
+ case IRP_MN_REMOVE_DEVICE:
+ __TargetRemoveDevice(Target);
+ break;
+
+ case IRP_MN_EJECT:
+ __TargetEject(Target);
+ break;
+
+ case IRP_MN_DEVICE_USAGE_NOTIFICATION:
+ __TargetDeviceUsageNotification(Target, Irp);
+ break;
+
+ default:
+ break;
+ }
+ TargetDereference(Target);
+ return DriverDispatchPnp(DeviceObject, Irp);
+}
+
+__drv_maxIRQL(DISPATCH_LEVEL)
+VOID
+TargetIssueDeviceEject(
+ __in PXENVBD_TARGET Target,
+ __in __nullterminated const CHAR* Reason
+ )
+{
+ KIRQL Irql;
+ BOOLEAN DoEject = FALSE;
+
+ KeAcquireSpinLock(&Target->Lock, &Irql);
+ if (Target->DeviceObject) {
+ DoEject = TRUE;
+ Target->EjectRequested = TRUE;
+ } else {
+ Target->EjectPending = TRUE;
+ }
+ KeReleaseSpinLock(&Target->Lock, Irql);
+
+ Verbose("Target[%d] : Ejecting (%s - %s)\n", TargetGetTargetId(Target), DoEject ? "Now" : "Next PnP IRP", Reason);
+ if (!Target->WrittenEjected) {
+ Target->WrittenEjected = TRUE;
+ FrontendStoreWriteFrontend(Target->Frontend, "ejected", "1");
+ }
+ if (DoEject) {
+ Verbose("Target[%d] : IoRequestDeviceEject(0x%p)\n", TargetGetTargetId(Target), Target->DeviceObject);
+ IoRequestDeviceEject(Target->DeviceObject);
+ } else {
+ Verbose("Target[%d] : Triggering BusChangeDetected to detect device\n", TargetGetTargetId(Target));
+ StorPortNotification(BusChangeDetected, TargetGetAdapter(Target), 0);
+ }
+}
+
+__checkReturn
+NTSTATUS
+TargetD3ToD0(
+ __in PXENVBD_TARGET Target
+ )
+{
+ NTSTATUS Status;
+ const ULONG TargetId = TargetGetTargetId(Target);
+
+ if (!TargetSetDevicePowerState(Target, PowerDeviceD0))
+ return STATUS_SUCCESS;
+
+ Trace("Target[%d] @ (%d) =====>\n", TargetId, KeGetCurrentIrql());
+ Verbose("Target[%d] : D3->D0\n", TargetId);
+
+ // power up frontend
+ Status = FrontendD3ToD0(Target->Frontend);
+ if (!NT_SUCCESS(Status))
+ goto fail1;
+
+ // connect frontend
+ Status = FrontendSetState(Target->Frontend, XENVBD_ENABLED);
+ if (!NT_SUCCESS(Status))
+ goto fail2;
+ __TargetUnpauseDataPath(Target);
+
+ Trace("Target[%d] @ (%d) <=====\n", TargetId, KeGetCurrentIrql());
+ return STATUS_SUCCESS;
+
+fail2:
+ Error("Fail2\n");
+ FrontendD0ToD3(Target->Frontend);
+
+fail1:
+ Error("Fail1 (%08x)\n", Status);
+
+ Target->DevicePowerState = PowerDeviceD3;
+
+ return Status;
+}
+
+VOID
+TargetD0ToD3(
+ __in PXENVBD_TARGET Target
+ )
+{
+ const ULONG TargetId = TargetGetTargetId(Target);
+
+ if (!TargetSetDevicePowerState(Target, PowerDeviceD3))
+ return;
+
+ Trace("Target[%d] @ (%d) =====>\n", TargetId, KeGetCurrentIrql());
+ Verbose("Target[%d] : D0->D3\n", TargetId);
+
+ // close frontend
+ __TargetPauseDataPath(Target, FALSE);
+ (VOID) FrontendSetState(Target->Frontend, XENVBD_CLOSED);
+ ASSERT3U(QueueCount(&Target->SubmittedReqs), ==, 0);
+
+ // power down frontend
+ FrontendD0ToD3(Target->Frontend);
+
+ Trace("Target[%d] @ (%d) <=====\n", TargetId, KeGetCurrentIrql());
+}
+
+__checkReturn
+BOOLEAN
+TargetCreate(
+ __in PXENVBD_ADAPTER Adapter,
+ __in __nullterminated PCHAR DeviceId,
+ __in ULONG TargetId,
+ __in XENVBD_DEVICE_TYPE DeviceType
+ )
+{
+ NTSTATUS Status;
+ PXENVBD_TARGET Target;
+
+ Trace("Target[%d] @ (%d) =====>\n", TargetId, KeGetCurrentIrql());
+
+ Status = STATUS_INSUFFICIENT_RESOURCES;
+#pragma warning(suppress: 6014)
+ Target = __TargetAlloc(sizeof(XENVBD_TARGET));
+ if (!Target)
+ goto fail1;
+
+ Verbose("Target[%d] : Creating\n", TargetId);
+ Target->Signature = TARGET_SIGNATURE;
+ Target->Adapter = Adapter;
+ Target->DeviceObject = NULL; // filled in later
+ KeInitializeEvent(&Target->RemoveEvent, SynchronizationEvent, FALSE);
+ Target->ReferenceCount = 1;
+ Target->Paused = 1; // Paused until D3->D0 transition
+ Target->DevicePnpState = Present;
+ Target->DevicePowerState = PowerDeviceD3;
+ Target->DeviceType = DeviceType;
+
+ KeInitializeSpinLock(&Target->Lock);
+ QueueInit(&Target->FreshSrbs);
+ QueueInit(&Target->PreparedReqs);
+ QueueInit(&Target->SubmittedReqs);
+ QueueInit(&Target->ShutdownSrbs);
+ __LookasideInit(&Target->RequestList, sizeof(XENVBD_REQUEST), REQUEST_POOL_TAG);
+ __LookasideInit(&Target->SegmentList, sizeof(XENVBD_SEGMENT), SEGMENT_POOL_TAG);
+ __LookasideInit(&Target->IndirectList, sizeof(XENVBD_INDIRECT), INDIRECT_POOL_TAG);
+
+ Status = FrontendCreate(Target, DeviceId, TargetId, &Target->Frontend);
+ if (!NT_SUCCESS(Status))
+ goto fail2;
+
+ Status = TargetD3ToD0(Target);
+ if (!NT_SUCCESS(Status))
+ goto fail3;
+
+ if (!AdapterLinkTarget(Adapter, Target))
+ goto fail4;
+
+ Verbose("Target[%d] : Created (%s)\n", TargetId, Target);
+ Trace("Target[%d] @ (%d) <=====\n", TargetId, KeGetCurrentIrql());
+ return TRUE;
+
+fail4:
+ Error("Fail4\n");
+ TargetD0ToD3(Target);
+
+fail3:
+ Error("Fail3\n");
+ FrontendDestroy(Target->Frontend);
+ Target->Frontend = NULL;
+
+fail2:
+ Error("Fail2\n");
+ __LookasideTerm(&Target->IndirectList);
+ __LookasideTerm(&Target->SegmentList);
+ __LookasideTerm(&Target->RequestList);
+ __TargetFree(Target);
+
+fail1:
+ Error("Fail1 (%08x)\n", Status);
+ return FALSE;
+}
+
+VOID
+TargetDestroy(
+ __in PXENVBD_TARGET Target
+ )
+{
+ const ULONG TargetId = TargetGetTargetId(Target);
+ PVOID Objects[4];
+ PKWAIT_BLOCK WaitBlock;
+
+ Trace("Target[%d] @ (%d) =====>\n", TargetId, KeGetCurrentIrql());
+ Verbose("Target[%d] : Destroying\n", TargetId);
+
+ ASSERT3U(Target->Signature, ==, TARGET_SIGNATURE);
+ if (!AdapterUnlinkTarget(TargetGetAdapter(Target), Target)) {
+ Error("Target[%d] : TARGET 0x%p not linked to ADAPTER 0x%p\n", TargetId, Target, TargetGetAdapter(Target));
+ }
+
+ TargetD0ToD3(Target);
+ TargetDereference(Target); // drop initial ref count
+
+ // Wait for ReferenceCount == 0 and RequestListUsed == 0
+ Verbose("Target[%d] : ReferenceCount %d, RequestListUsed %d\n", TargetId, Target->ReferenceCount, Target->RequestList.Used);
+ Objects[0] = &Target->RemoveEvent;
+ Objects[1] = &Target->RequestList.Empty;
+ Objects[2] = &Target->SegmentList.Empty;
+ Objects[3] = &Target->IndirectList.Empty;
+
+ WaitBlock = (PKWAIT_BLOCK)__TargetAlloc(sizeof(KWAIT_BLOCK) * ARRAYSIZE(Objects));
+ if (WaitBlock == NULL) {
+ ULONG Index;
+
+ Error("Unable to allocate resources for KWAIT_BLOCK\n");
+
+ for (Index = 0; Index < ARRAYSIZE(Objects); Index++)
+ KeWaitForSingleObject(Objects[Index],
+ Executive,
+ KernelMode,
+ FALSE,
+ NULL);
+ } else {
+ KeWaitForMultipleObjects(ARRAYSIZE(Objects),
+ Objects,
+ WaitAll,
+ Executive,
+ KernelMode,
+ FALSE,
+ NULL,
+ WaitBlock);
+#pragma prefast(suppress:6102)
+ __TargetFree(WaitBlock);
+ }
+
+ ASSERT3S(Target->ReferenceCount, ==, 0);
+ ASSERT3U(TargetGetDevicePnpState(Target), ==, Deleted);
+
+ FrontendDestroy(Target->Frontend);
+ Target->Frontend = NULL;
+
+ __LookasideTerm(&Target->IndirectList);
+ __LookasideTerm(&Target->SegmentList);
+ __LookasideTerm(&Target->RequestList);
+
+ ASSERT3U(Target->Signature, ==, TARGET_SIGNATURE);
+ RtlZeroMemory(Target, sizeof(XENVBD_TARGET));
+ __TargetFree(Target);
+
+ Verbose("Target[%d] : Destroyed\n", TargetId);
+ Trace("Target[%d] @ (%d) <=====\n", TargetId, KeGetCurrentIrql());
+}
--- /dev/null
+/* Copyright (c) Citrix Systems Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms,
+ * with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer in the documentation and/or other
+ * materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _XENVBD_TARGET_H
+#define _XENVBD_TARGET_H
+
+typedef struct _XENVBD_TARGET XENVBD_TARGET, *PXENVBD_TARGET;
+
+#include <ntddk.h>
+#include <ntstrsafe.h>
+#include <xenvbd-storport.h>
+#include "adapter.h"
+#include "srbext.h"
+#include "types.h"
+#include <debug_interface.h>
+
+extern VOID
+TargetDebugCallback(
+ __in PXENVBD_TARGET Target,
+ __in PXENBUS_DEBUG_INTERFACE Debug
+ );
+
+// Creation/Deletion
+__checkReturn
+extern BOOLEAN
+TargetCreate(
+ __in PXENVBD_ADAPTER Adapter,
+ __in __nullterminated PCHAR DeviceId,
+ __in ULONG TargetId,
+ __in XENVBD_DEVICE_TYPE DeviceType
+ );
+
+extern VOID
+TargetDestroy(
+ __in PXENVBD_TARGET Target
+ );
+
+__checkReturn
+extern NTSTATUS
+TargetD3ToD0(
+ __in PXENVBD_TARGET Target
+ );
+
+extern VOID
+TargetD0ToD3(
+ __in PXENVBD_TARGET Target
+ );
+
+// PnP States
+extern VOID
+TargetSetMissing(
+ __in PXENVBD_TARGET Target,
+ __in __nullterminated const CHAR* Reason
+ );
+
+__checkReturn
+extern BOOLEAN
+TargetIsMissing(
+ __in PXENVBD_TARGET Target
+ );
+
+extern const CHAR*
+TargetMissingReason(
+ __in PXENVBD_TARGET Target
+ );
+
+__checkReturn
+extern BOOLEAN
+TargetIsEmulatedUnplugged(
+ __in PXENVBD_TARGET Target
+ );
+
+extern VOID
+TargetSetDevicePnpState(
+ __in PXENVBD_TARGET Target,
+ __in DEVICE_PNP_STATE State
+ );
+
+__checkReturn
+extern DEVICE_PNP_STATE
+TargetGetDevicePnpState(
+ __in PXENVBD_TARGET Target
+ );
+
+// Reference Counting
+extern LONG
+__TargetReference(
+ __in PXENVBD_TARGET Target,
+ __in PCHAR Caller
+ );
+
+#define TargetReference(_x_) __TargetReference(_x_, __FUNCTION__)
+
+extern LONG
+__TargetDereference(
+ __in PXENVBD_TARGET Target,
+ __in PCHAR Caller
+ );
+
+#define TargetDereference(_x_) __TargetDereference(_x_, __FUNCTION__)
+
+// Query Methods
+extern ULONG
+TargetGetTargetId(
+ __in PXENVBD_TARGET Target
+ );
+
+__checkReturn
+extern PDEVICE_OBJECT
+TargetGetDeviceObject(
+ __in PXENVBD_TARGET Target
+ );
+
+extern VOID
+TargetSetDeviceObject(
+ __in PXENVBD_TARGET Target,
+ __in PDEVICE_OBJECT DeviceObject
+ );
+
+__checkReturn
+extern BOOLEAN
+TargetIsPaused(
+ __in PXENVBD_TARGET Target
+ );
+
+__checkReturn
+extern ULONG
+TargetOutstandingReqs(
+ __in PXENVBD_TARGET Target
+ );
+
+__checkReturn
+extern PXENVBD_ADAPTER
+TargetGetAdapter(
+ __in PXENVBD_TARGET Target
+ );
+
+extern ULONG
+TargetSectorSize(
+ __in PXENVBD_TARGET Target
+ );
+
+// Queue-Related
+extern VOID
+TargetSubmitRequests(
+ __in PXENVBD_TARGET Target
+ );
+
+extern VOID
+TargetCompleteResponse(
+ __in PXENVBD_TARGET Target,
+ __in ULONG Tag,
+ __in SHORT Status
+ );
+
+extern VOID
+TargetPreResume(
+ __in PXENVBD_TARGET Target
+ );
+
+extern VOID
+TargetPostResume(
+ __in PXENVBD_TARGET Target
+ );
+
+// StorPort Methods
+extern VOID
+TargetReset(
+ __in PXENVBD_TARGET Target
+ );
+
+__checkReturn
+extern BOOLEAN
+TargetStartIo(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_REQUEST_BLOCK Srb
+ );
+
+extern VOID
+TargetSrbPnp(
+ __in PXENVBD_TARGET Target,
+ __in PSCSI_PNP_REQUEST_BLOCK Srb
+ );
+
+// PnP Handler
+__checkReturn
+extern NTSTATUS
+TargetDispatchPnp(
+ __in PXENVBD_TARGET Target,
+ __in PDEVICE_OBJECT DeviceObject,
+ __in PIRP Irp
+ );
+
+__drv_maxIRQL(DISPATCH_LEVEL)
+extern VOID
+TargetIssueDeviceEject(
+ __in PXENVBD_TARGET Target,
+ __in __nullterminated const CHAR* Reason
+ );
+
+#endif // _XENVBD_TARGET_H
<ClCompile Include="../../src/xenvbd/registry.c" />
<ClCompile Include="../../src/xenvbd/adapter.c" />
<ClCompile Include="../../src/xenvbd/frontend.c" />
- <ClCompile Include="../../src/xenvbd/pdo.c" />
+ <ClCompile Include="../../src/xenvbd/target.c" />
<ClCompile Include="../../src/xenvbd/pdoinquiry.c" />
<ClCompile Include="../../src/xenvbd/queue.c" />
<ClCompile Include="../../src/xenvbd/thread.c" />
projects / pvdrivers / win / xenvbd.git / commitdiff