From: Owen Smith Date: Thu, 4 Sep 2014 08:19:40 +0000 (+0100) Subject: Fix BLKIF_OP_INDIRECT protocol requests X-Git-Tag: 8.1.0-rc1~44 X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=0a4f7052d3acf7817a99d46a68f8ab40c3f69812;p=pvdrivers%2Fwin%2Fxenvbd.git Fix BLKIF_OP_INDIRECT protocol requests Fixes an issue with indirect requests that arose due to changes in XenBus's (7.x) grant table interface (v3 to v4). Refactored for the 8.0 version of XenBus. Rework internal request structures to contain a list of segments. Number of segments determines the difference between direct and indirect operations. blkback is limited to 256 segments per indirect request, XenVbd reports support for 175 physical breaks (i.e. segments). Signed-off-by: Owen Smith --- diff --git a/src/xenvbd/blockring.c b/src/xenvbd/blockring.c index b0782d3..35ffb3b 100644 --- a/src/xenvbd/blockring.c +++ b/src/xenvbd/blockring.c @@ -146,22 +146,68 @@ __BlockRingInsert( ) { PXENVBD_GRANTER Granter = FrontendGetGranter(BlockRing->Frontend); - ULONG Index; - blkif_request_discard_t* req_discard; - blkif_request_indirect_t* req_indirect; switch (Request->Operation) { case BLKIF_OP_READ: case BLKIF_OP_WRITE: - req->operation = Request->Operation; - req->nr_segments = Request->u.ReadWrite.NrSegments; - req->handle = (USHORT)BlockRing->DeviceId; - req->id = __BlockRingGetTag(BlockRing, Request); - req->sector_number = Request->u.ReadWrite.FirstSector; - for (Index = 0; Index < Request->u.ReadWrite.NrSegments; ++Index) { - req->seg[Index].gref = GranterReference(Granter, Request->u.ReadWrite.Segments[Index].Grant); - req->seg[Index].first_sect = Request->u.ReadWrite.Segments[Index].FirstSector; - req->seg[Index].last_sect = Request->u.ReadWrite.Segments[Index].LastSector; + if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { + // Indirect + ULONG PageIndex; + ULONG SegmentIndex; + PLIST_ENTRY Entry; + blkif_request_indirect_t* req_indirect; + + req_indirect = (blkif_request_indirect_t*)req; + req_indirect->operation = BLKIF_OP_INDIRECT; + req_indirect->indirect_op = Request->Operation; + req_indirect->nr_segments = Request->NrSegments; + req_indirect->id = __BlockRingGetTag(BlockRing, Request); + req_indirect->sector_number = Request->FirstSector; + req_indirect->handle = (USHORT)BlockRing->DeviceId; + + for (PageIndex = 0, SegmentIndex = 0, Entry = Request->Segments.Flink; + PageIndex < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && + Entry != &Request->Segments; + Entry = Entry->Flink) { + PBLKIF_SEGMENT Indirect = Request->Pages[PageIndex]; + PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry); + + Indirect[SegmentIndex].GrantRef = GranterReference(Granter, Segment->Grant); + Indirect[SegmentIndex].First = Segment->FirstSector; + Indirect[SegmentIndex].Last = Segment->LastSector; + + ++SegmentIndex; + if (SegmentIndex >= XENVBD_MAX_SEGMENTS_PER_PAGE) { + ++PageIndex; + SegmentIndex = 0; + } + } + for (PageIndex = 0; + PageIndex < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && + Request->Grants[PageIndex] != NULL; + ++PageIndex) { + req_indirect->indirect_grefs[PageIndex] = GranterReference(Granter, Request->Grants[PageIndex]); + } + } else { + // Direct + ULONG Index; + PLIST_ENTRY Entry; + + req->operation = Request->Operation; + req->nr_segments = (UCHAR)Request->NrSegments; + req->handle = (USHORT)BlockRing->DeviceId; + req->id = __BlockRingGetTag(BlockRing, Request); + req->sector_number = Request->FirstSector; + + for (Index = 0, Entry = Request->Segments.Flink; + Index < BLKIF_MAX_SEGMENTS_PER_REQUEST && + Entry != &Request->Segments; + ++Index, Entry = Entry->Flink) { + PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry); + req->seg[Index].gref = GranterReference(Granter, Segment->Grant); + req->seg[Index].first_sect = Segment->FirstSector; + req->seg[Index].last_sect = Segment->LastSector; + } } break; @@ -170,31 +216,19 @@ __BlockRingInsert( req->nr_segments = 0; req->handle = (USHORT)BlockRing->DeviceId; req->id = __BlockRingGetTag(BlockRing, Request); - req->sector_number = Request->u.Barrier.FirstSector; + req->sector_number = Request->FirstSector; break; - case BLKIF_OP_DISCARD: + case BLKIF_OP_DISCARD: { + blkif_request_discard_t* req_discard; req_discard = (blkif_request_discard_t*)req; req_discard->operation = BLKIF_OP_DISCARD; - req_discard->flag = Request->u.Discard.Flags; + req_discard->flag = Request->Flags; req_discard->handle = (USHORT)BlockRing->DeviceId; req_discard->id = __BlockRingGetTag(BlockRing, Request); - req_discard->sector_number = Request->u.Discard.FirstSector; - req_discard->nr_sectors = Request->u.Discard.NrSectors; - break; - - case BLKIF_OP_INDIRECT: - req_indirect = (blkif_request_indirect_t*)req; - req_indirect->operation = BLKIF_OP_INDIRECT; - req_indirect->indirect_op = Request->u.Indirect.Operation; - req_indirect->nr_segments = Request->u.Indirect.NrSegments; - req_indirect->id = __BlockRingGetTag(BlockRing, Request); - req_indirect->sector_number = Request->u.Indirect.FirstSector; - req_indirect->handle = (USHORT)BlockRing->DeviceId; - for (Index = 0; Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST; ++Index) { - req_indirect->indirect_grefs[Index] = GranterReference(Granter, Request->u.Indirect.Grants[Index]); - } - break; + req_discard->sector_number = Request->FirstSector; + req_discard->nr_sectors = Request->NrSectors; + } break; default: ASSERT(FALSE); diff --git a/src/xenvbd/pdo.c b/src/xenvbd/pdo.c index c7eaa03..9762944 100644 --- a/src/xenvbd/pdo.c +++ b/src/xenvbd/pdo.c @@ -98,9 +98,9 @@ struct _XENVBD_PDO { const CHAR* Reason; // SRBs - XENVBD_LOOKASIDE SegmentList; - XENVBD_LOOKASIDE MappingList; XENVBD_LOOKASIDE RequestList; + XENVBD_LOOKASIDE SegmentList; + XENVBD_LOOKASIDE IndirectList; XENVBD_QUEUE FreshSrbs; XENVBD_QUEUE PreparedReqs; XENVBD_QUEUE SubmittedReqs; @@ -127,10 +127,7 @@ struct _XENVBD_PDO { #define PDO_POOL_TAG 'odPX' #define REQUEST_POOL_TAG 'qeRX' #define SEGMENT_POOL_TAG 'geSX' -#define MAPPING_POOL_TAG 'paMX' -#define SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(XENVBD_SEGMENT)) -#define SEGMENT_LIST_SIZE (SEGMENTS_PER_PAGE * sizeof(XENVBD_SEGMENT)) -#define MAPPING_LIST_SIZE (SEGMENTS_PER_PAGE * sizeof(XENVBD_MAPPING)) +#define INDIRECT_POOL_TAG 'dnIX' __checkReturn __drv_allocatesMem(mem) @@ -310,7 +307,7 @@ PdoDebugCallback( __LookasideDebug(&Pdo->RequestList, DebugInterface, "REQUESTs"); __LookasideDebug(&Pdo->SegmentList, DebugInterface, "SEGMENTs"); - __LookasideDebug(&Pdo->MappingList, DebugInterface, "MAPPINGs"); + __LookasideDebug(&Pdo->IndirectList, DebugInterface, "INDIRECTs"); QueueDebugCallback(&Pdo->FreshSrbs, "Fresh ", DebugInterface); QueueDebugCallback(&Pdo->PreparedReqs, "Prepared ", DebugInterface); @@ -541,8 +538,8 @@ PdoCreate( goto fail2; __LookasideInit(&Pdo->RequestList, sizeof(XENVBD_REQUEST), REQUEST_POOL_TAG); - __LookasideInit(&Pdo->SegmentList, SEGMENT_LIST_SIZE, SEGMENT_POOL_TAG); - __LookasideInit(&Pdo->MappingList, MAPPING_LIST_SIZE, MAPPING_POOL_TAG); + __LookasideInit(&Pdo->SegmentList, sizeof(XENVBD_SEGMENT), SEGMENT_POOL_TAG); + __LookasideInit(&Pdo->IndirectList, PAGE_SIZE, INDIRECT_POOL_TAG); Status = PdoD3ToD0(Pdo); if (!NT_SUCCESS(Status)) @@ -561,7 +558,7 @@ fail4: fail3: Error("Fail3\n"); - __LookasideTerm(&Pdo->MappingList); + __LookasideTerm(&Pdo->IndirectList); __LookasideTerm(&Pdo->SegmentList); __LookasideTerm(&Pdo->RequestList); FrontendDestroy(Pdo->Frontend); @@ -601,7 +598,7 @@ PdoDestroy( Objects[0] = &Pdo->RemoveEvent; Objects[1] = &Pdo->RequestList.Empty; Objects[2] = &Pdo->SegmentList.Empty; - Objects[3] = &Pdo->MappingList.Empty; + Objects[3] = &Pdo->IndirectList.Empty; WaitBlock = (PKWAIT_BLOCK)__PdoAlloc(sizeof(KWAIT_BLOCK) * ARRAYSIZE(Objects)); if (WaitBlock == NULL) { @@ -630,7 +627,7 @@ PdoDestroy( ASSERT3S(Pdo->ReferenceCount, ==, 0); ASSERT3U(PdoGetDevicePnpState(Pdo), ==, Deleted); - __LookasideTerm(&Pdo->MappingList); + __LookasideTerm(&Pdo->IndirectList); __LookasideTerm(&Pdo->SegmentList); __LookasideTerm(&Pdo->RequestList); @@ -887,18 +884,27 @@ __PdoIncBlkifOpCount( ) { switch (Request->Operation) { - case BLKIF_OP_READ: ++Pdo->BlkOpRead; break; - case BLKIF_OP_WRITE: ++Pdo->BlkOpWrite; break; - case BLKIF_OP_WRITE_BARRIER: ++Pdo->BlkOpBarrier; break; - case BLKIF_OP_DISCARD: ++Pdo->BlkOpDiscard; break; - case BLKIF_OP_INDIRECT: - switch (Request->u.Indirect.Operation) { - case BLKIF_OP_READ: ++Pdo->BlkOpIndirectRead; break; - case BLKIF_OP_WRITE: ++Pdo->BlkOpIndirectWrite; break; - default: ASSERT(FALSE); break; - } + case BLKIF_OP_READ: + if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) + ++Pdo->BlkOpIndirectRead; + else + ++Pdo->BlkOpRead; + break; + case BLKIF_OP_WRITE: + if (Request->NrSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) + ++Pdo->BlkOpIndirectWrite; + else + ++Pdo->BlkOpWrite; + break; + case BLKIF_OP_WRITE_BARRIER: + ++Pdo->BlkOpBarrier; + break; + case BLKIF_OP_DISCARD: + ++Pdo->BlkOpDiscard; + break; + default: + ASSERT(FALSE); break; - default: ASSERT(FALSE); break; } } @@ -1009,7 +1015,7 @@ SGListNext( static FORCEINLINE BOOLEAN MapSegmentBuffer( IN PXENVBD_PDO Pdo, - IN PXENVBD_MAPPING Mapping, + IN PXENVBD_SEGMENT Segment, IN PXENVBD_SG_LIST SGList, IN ULONG SectorSize, IN ULONG SectorsNow @@ -1020,7 +1026,7 @@ MapSegmentBuffer( // map PhysAddr to 1 or 2 pages and lock for VirtAddr #pragma warning(push) #pragma warning(disable:28145) - Mdl = &Mapping->Mdl; + Mdl = &Segment->Mdl; Mdl->Next = NULL; Mdl->Size = (SHORT)(sizeof(MDL) + sizeof(PFN_NUMBER)); Mdl->MdlFlags = MDL_PAGES_LOCKED; @@ -1029,13 +1035,13 @@ MapSegmentBuffer( Mdl->StartVa = NULL; Mdl->ByteCount = SGList->PhysLen; Mdl->ByteOffset = __Offset(SGList->PhysAddr); - Mapping->Pfn[0] = __Phys2Pfn(SGList->PhysAddr); + Segment->Pfn[0] = __Phys2Pfn(SGList->PhysAddr); if (SGList->PhysLen < SectorsNow * SectorSize) { SGListGet(SGList); Mdl->Size += sizeof(PFN_NUMBER); Mdl->ByteCount = Mdl->ByteCount + SGList->PhysLen; - Mapping->Pfn[1] = __Phys2Pfn(SGList->PhysAddr); + Segment->Pfn[1] = __Phys2Pfn(SGList->PhysAddr); } #pragma warning(pop) @@ -1043,15 +1049,15 @@ MapSegmentBuffer( ASSERT3U(Mdl->ByteCount, <=, PAGE_SIZE); ASSERT3U(SectorsNow, ==, (Mdl->ByteCount / SectorSize)); - Mapping->Length = __min(Mdl->ByteCount, PAGE_SIZE); - Mapping->Buffer = MmMapLockedPagesSpecifyCache(Mdl, KernelMode, + Segment->Length = __min(Mdl->ByteCount, PAGE_SIZE); + Segment->Buffer = MmMapLockedPagesSpecifyCache(Mdl, KernelMode, MmCached, NULL, FALSE, __PdoPriority(Pdo)); - if (!Mapping->Buffer) { + if (!Segment->Buffer) { goto fail; } - ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Mapping->Pfn[0]); - ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Mapping->Pfn[1]); + ASSERT3P(MmGetMdlPfnArray(Mdl)[0], ==, Segment->Pfn[0]); + ASSERT3P(MmGetMdlPfnArray(Mdl)[1], ==, Segment->Pfn[1]); return TRUE; @@ -1061,14 +1067,14 @@ fail: static FORCEINLINE VOID UnmapSegmentBuffer( - IN PXENVBD_MAPPING Mapping + IN PXENVBD_SEGMENT Segment ) { - MmUnmapLockedPages(Mapping->Buffer, &Mapping->Mdl); - RtlZeroMemory(&Mapping->Mdl, sizeof(Mapping->Mdl)); - RtlZeroMemory(Mapping->Pfn, sizeof(Mapping->Pfn)); - Mapping->Buffer = NULL; - Mapping->Length = 0; + MmUnmapLockedPages(Segment->Buffer, &Segment->Mdl); + RtlZeroMemory(&Segment->Mdl, sizeof(Segment->Mdl)); + RtlZeroMemory(Segment->Pfn, sizeof(Segment->Pfn)); + Segment->Buffer = NULL; + Segment->Length = 0; } static FORCEINLINE VOID @@ -1077,68 +1083,42 @@ RequestCleanup( IN PXENVBD_REQUEST Request ) { - ULONG Index, Index2; + ULONG Index; PXENVBD_GRANTER Granter = FrontendGetGranter(Pdo->Frontend); - // cleanup granted/mapped buffers - switch (Request->Operation) { - case BLKIF_OP_READ: - case BLKIF_OP_WRITE: - for (Index = 0; Index < XENVBD_MAX_SEGMENTS_PER_REQUEST; ++Index) { - PXENVBD_SEGMENT Segment = &Request->u.ReadWrite.Segments[Index]; - if (Segment->Grant) - GranterPut(Granter, Segment->Grant); - Segment->Grant = NULL; - } - for (Index = 0; Index < XENVBD_MAX_SEGMENTS_PER_REQUEST; ++Index) { - PXENVBD_MAPPING Mapping = &Request->u.ReadWrite.Mappings[Index]; - if (Mapping->BufferId) - BufferPut(Mapping->BufferId); - Mapping->BufferId = NULL; - if (Mapping->Buffer) - UnmapSegmentBuffer(Mapping); - } - break; + for (;;) { + PLIST_ENTRY Entry; + PXENVBD_SEGMENT Segment; - case BLKIF_OP_INDIRECT: - for (Index = 0; Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST; ++Index) { - if (Request->u.Indirect.Grants[Index]) - GranterPut(Granter, Request->u.Indirect.Grants[Index]); - Request->u.Indirect.Grants[Index] = 0; - } - for (Index = 0; Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST; ++Index) { - PXENVBD_SEGMENT SegmentList = Request->u.Indirect.Segments[Index]; - if (SegmentList == NULL) - continue; - for (Index2 = 0; Index2 < SEGMENTS_PER_PAGE; ++Index2) { - PXENVBD_SEGMENT Segment = &SegmentList[Index2]; - if (Segment->Grant) - GranterPut(Granter, Segment->Grant); - Segment->Grant = NULL; - } - __LookasideFree(&Pdo->SegmentList, SegmentList); - Request->u.Indirect.Segments[Index] = NULL; - } - for (Index = 0; Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST; ++Index) { - PXENVBD_MAPPING MappingList = Request->u.Indirect.Mappings[Index]; - if (MappingList == NULL) - continue; - for (Index2 = 0; Index2 < SEGMENTS_PER_PAGE; ++Index2) { - PXENVBD_MAPPING Mapping = &MappingList[Index2]; - if (Mapping->BufferId) - BufferPut(Mapping->BufferId); - Mapping->BufferId = NULL; - if (Mapping->Buffer) - UnmapSegmentBuffer(Mapping); - } - __LookasideFree(&Pdo->MappingList, MappingList); - Request->u.Indirect.Mappings[Index] = NULL; - } - break; + Entry = RemoveHeadList(&Request->Segments); + if (Entry == &Request->Segments) + break; - default: - // no special cleanup - break; + Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry); + --Request->NrSegments; + + if (Segment->Grant) + GranterPut(Granter, Segment->Grant); + Segment->Grant = NULL; + + if (Segment->BufferId) + BufferPut(Segment->BufferId); + Segment->BufferId = NULL; + + if (Segment->Buffer) + UnmapSegmentBuffer(Segment); + + __LookasideFree(&Pdo->SegmentList, Segment); + } + + for (Index = 0; Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST; ++Index) { + if (Request->Grants[Index]) + GranterPut(Granter, Request->Grants[Index]); + Request->Grants[Index] = NULL; + + if (Request->Pages[Index]) + __LookasideFree(&Pdo->IndirectList, Request->Pages[Index]); + Request->Pages[Index] = NULL; } } @@ -1147,50 +1127,18 @@ RequestCopyOutput( __in PXENVBD_REQUEST Request ) { - ULONG Index, Index2; - ULONG NrSegments; + PLIST_ENTRY Entry; - switch (Request->Operation) { - case BLKIF_OP_READ: - NrSegments = Request->u.ReadWrite.NrSegments; - for (Index = 0; - Index < BLKIF_MAX_SEGMENTS_PER_REQUEST && - NrSegments > 0; - ++Index, --NrSegments) { - PXENVBD_MAPPING Mapping = &Request->u.ReadWrite.Mappings[Index]; - - if (Mapping->BufferId) - BufferCopyOut(Mapping->BufferId, Mapping->Buffer, Mapping->Length); - } - break; + if (Request->Operation != BLKIF_OP_READ) + return; - case BLKIF_OP_INDIRECT: - if (Request->u.Indirect.Operation != BLKIF_OP_READ) - break; // not an INDIRECT READ, dont copy any data - - NrSegments = Request->u.Indirect.NrSegments; - for (Index = 0; - Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && - NrSegments > 0; - ++Index) { - PXENVBD_MAPPING MappingList = Request->u.Indirect.Mappings[Index]; - if (MappingList == NULL) - continue; - for (Index2 = 0; - Index2 < SEGMENTS_PER_PAGE && - NrSegments > 0; - ++Index2, --NrSegments) { - PXENVBD_MAPPING Mapping = &MappingList[Index2]; - - if (Mapping->BufferId) - BufferCopyOut(Mapping->BufferId, Mapping->Buffer, Mapping->Length); - } - } - break; + for (Entry = Request->Segments.Flink; + Entry != &Request->Segments; + Entry = Entry->Flink) { + PXENVBD_SEGMENT Segment = CONTAINING_RECORD(Entry, XENVBD_SEGMENT, Entry); - default: - // not a READ, dont copy any data - break; + if (Segment->BufferId) + BufferCopyOut(Segment->BufferId, Segment->Buffer, Segment->Length); } } @@ -1198,7 +1146,6 @@ static NTSTATUS PrepareSegment( IN PXENVBD_PDO Pdo, IN PXENVBD_SEGMENT Segment, - IN PXENVBD_MAPPING Mapping, IN PXENVBD_SG_LIST SGList, IN BOOLEAN ReadOnly, IN ULONG SectorsLeft, @@ -1217,9 +1164,9 @@ PrepareSegment( Segment->FirstSector = (UCHAR)((__Offset(SGList->PhysAddr) + SectorSize - 1) / SectorSize); *SectorsNow = __min(SectorsLeft, SectorsPerPage - Segment->FirstSector); Segment->LastSector = (UCHAR)(Segment->FirstSector + *SectorsNow - 1); - Mapping->BufferId = NULL; // granted, ensure its null - Mapping->Buffer = NULL; // granted, ensure its null - Mapping->Length = 0; // granted, ensure its 0 + Segment->BufferId = NULL; // granted, ensure its null + Segment->Buffer = NULL; // granted, ensure its null + Segment->Length = 0; // granted, ensure its 0 Pfn = __Phys2Pfn(SGList->PhysAddr); ASSERT3U((SGList->PhysLen / SectorSize), ==, *SectorsNow); @@ -1232,20 +1179,20 @@ PrepareSegment( Segment->LastSector = (UCHAR)(*SectorsNow - 1); // map SGList to Virtual Address. Populates Segment->Buffer and Segment->Length - if (!MapSegmentBuffer(Pdo, Mapping, SGList, SectorSize, *SectorsNow)) { + if (!MapSegmentBuffer(Pdo, Segment, SGList, SectorSize, *SectorsNow)) { ++Pdo->FailedMaps; goto fail; } // get a buffer - if (!BufferGet(Segment, &Mapping->BufferId, &Pfn)) { + if (!BufferGet(Segment, &Segment->BufferId, &Pfn)) { ++Pdo->FailedBounces; goto fail; } // copy contents in if (ReadOnly) { // Operation == BLKIF_OP_WRITE - BufferCopyIn(Mapping->BufferId, Mapping->Buffer, Mapping->Length); + BufferCopyIn(Segment->BufferId, Segment->Buffer, Segment->Length); } } @@ -1267,6 +1214,7 @@ PrepareBlkifReadWrite( IN PXENVBD_PDO Pdo, IN PXENVBD_REQUEST Request, IN PXENVBD_SG_LIST SGList, + IN ULONG MaxSegments, IN ULONG64 SectorStart, IN ULONG SectorsLeft, OUT PULONG SectorsDone @@ -1279,138 +1227,100 @@ PrepareBlkifReadWrite( __Operation(Cdb_OperationEx(Request->Srb), &Operation, &ReadOnly); Request->Operation = Operation; - Request->u.ReadWrite.NrSegments = 0; - Request->u.ReadWrite.FirstSector = SectorStart; + Request->NrSegments = 0; + Request->FirstSector = SectorStart; for (Index = 0; - Index < BLKIF_MAX_SEGMENTS_PER_REQUEST && + Index < MaxSegments && SectorsLeft > 0; ++Index) { - PXENVBD_SEGMENT Segment = &Request->u.ReadWrite.Segments[Index]; - PXENVBD_MAPPING Mapping = &Request->u.ReadWrite.Mappings[Index]; + PXENVBD_SEGMENT Segment; ULONG SectorsNow; - Request->u.ReadWrite.NrSegments++; + Segment = __LookasideAlloc(&Pdo->SegmentList); + Status = STATUS_NO_MEMORY; + if (Segment == NULL) + goto fail1; + + InsertTailList(&Request->Segments, &Segment->Entry); + ++Request->NrSegments; + Status = PrepareSegment(Pdo, Segment, - Mapping, SGList, ReadOnly, SectorsLeft, &SectorsNow); if (!NT_SUCCESS(Status)) - goto fail; + goto fail2; *SectorsDone += SectorsNow; SectorsLeft -= SectorsNow; } - ASSERT3U(Request->u.ReadWrite.NrSegments, >, 0); - ASSERT3U(Request->u.ReadWrite.NrSegments, <=, BLKIF_MAX_SEGMENTS_PER_REQUEST); + ASSERT3U(Request->NrSegments, >, 0); + ASSERT3U(Request->NrSegments, <=, MaxSegments); return STATUS_SUCCESS; -fail: +fail2: +fail1: return Status; } static NTSTATUS PrepareBlkifIndirect( IN PXENVBD_PDO Pdo, - IN PXENVBD_REQUEST Request, - IN PXENVBD_SG_LIST SGList, - IN ULONG64 SectorStart, - IN ULONG SectorsLeft, - OUT PULONG SectorsDone + IN PXENVBD_REQUEST Request ) { - ULONG Index, Index2; - UCHAR Operation; - BOOLEAN ReadOnly; - NTSTATUS Status; + NTSTATUS status; + ULONG Index; + ULONG NrSegments = 0; PXENVBD_GRANTER Granter = FrontendGetGranter(Pdo->Frontend); - const ULONG MaxSegments = FrontendGetFeatures(Pdo->Frontend)->Indirect; - __Operation(Cdb_OperationEx(Request->Srb), &Operation, &ReadOnly); - - Request->Operation = BLKIF_OP_INDIRECT; - Request->u.Indirect.Operation = Operation; - Request->u.Indirect.NrSegments = 0; - Request->u.Indirect.FirstSector = SectorStart; for (Index = 0; - Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && - SectorsLeft > 0 && - Request->u.Indirect.NrSegments < MaxSegments; - ++Index) { - PXENVBD_SEGMENT SegmentList; - PXENVBD_MAPPING MappingList; + Index < BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST && + NrSegments < Request->NrSegments; + ++Index) { + Request->Pages[Index] = __LookasideAlloc(&Pdo->IndirectList); - Status = STATUS_NO_MEMORY; - Request->u.Indirect.Segments[Index] = SegmentList = __LookasideAlloc(&Pdo->SegmentList); - if (SegmentList == NULL) - goto fail; - - Status = STATUS_NO_MEMORY; - Request->u.Indirect.Mappings[Index] = MappingList = __LookasideAlloc(&Pdo->MappingList); - if (MappingList == NULL) - goto fail; + status = STATUS_NO_MEMORY; + if (Request->Pages[Index] == NULL) + goto fail1; - for (Index2 = 0; - Index2 < SEGMENTS_PER_PAGE && - SectorsLeft > 0 && - Request->u.Indirect.NrSegments < MaxSegments; - ++Index2) { - PXENVBD_SEGMENT Segment = &SegmentList[Index2]; - PXENVBD_MAPPING Mapping = &MappingList[Index2]; - ULONG SectorsNow = 0; - - Request->u.Indirect.NrSegments++; - Status = PrepareSegment(Pdo, - Segment, - Mapping, - SGList, - ReadOnly, - SectorsLeft, - &SectorsNow); - if(!NT_SUCCESS(Status)) - goto fail; - - *SectorsDone += SectorsNow; - SectorsLeft -= SectorsNow; - } - - Status = GranterGet(Granter, - __Virt2Pfn(SegmentList), + status = GranterGet(Granter, + __Virt2Pfn(Request->Pages[Index]), TRUE, - &Request->u.Indirect.Grants[Index]); - if (!NT_SUCCESS(Status)) { - ++Pdo->FailedGrants; - goto fail; - } + &Request->Grants[Index]); + if (!NT_SUCCESS(status)) + goto fail2; + + NrSegments += XENVBD_MAX_SEGMENTS_PER_PAGE; } - ASSERT3U(Request->u.Indirect.NrSegments, >, 0); - ASSERT3U(Request->u.Indirect.NrSegments, <=, MaxSegments); return STATUS_SUCCESS; -fail: - return Status; +fail2: +fail1: + return status; } -static FORCEINLINE BOOLEAN +static FORCEINLINE ULONG UseIndirect( IN PXENVBD_PDO Pdo, IN ULONG SectorsLeft ) { const ULONG SectorsPerPage = __SectorsPerPage(PdoSectorSize(Pdo)); + const ULONG MaxIndirectSegs = FrontendGetFeatures(Pdo->Frontend)->Indirect; - if (FrontendGetFeatures(Pdo->Frontend)->Indirect == 0) - return FALSE; // not supported + if (MaxIndirectSegs <= BLKIF_MAX_SEGMENTS_PER_REQUEST) + return BLKIF_MAX_SEGMENTS_PER_REQUEST; // not supported if (SectorsLeft < BLKIF_MAX_SEGMENTS_PER_REQUEST * SectorsPerPage) - return FALSE; // first into a single BLKIF_OP_{READ/WRITE} + return BLKIF_MAX_SEGMENTS_PER_REQUEST; // first into a single BLKIF_OP_{READ/WRITE} - return TRUE; + return MaxIndirectSegs; } __checkReturn @@ -1436,6 +1346,7 @@ PrepareReadWrite( Srb->SrbStatus = SRB_STATUS_PENDING; while (SectorsLeft > 0) { + ULONG MaxSegments; ULONG SectorsDone = 0; PXENVBD_REQUEST Request = __LookasideAlloc(&Pdo->RequestList); @@ -1445,27 +1356,28 @@ PrepareReadWrite( Request->Srb = Srb; Request->Id = PdoGetTag(Pdo); + InitializeListHead(&Request->Segments); InsertTailList(&ReqList, &Request->Entry); - InterlockedIncrement(&SrbExt->Count); - if (UseIndirect(Pdo, SectorsLeft)) { - Status = PrepareBlkifIndirect(Pdo, - Request, - &SGList, - SectorStart, - SectorsLeft, - &SectorsDone); - } else { - Status = PrepareBlkifReadWrite(Pdo, - Request, - &SGList, - SectorStart, - SectorsLeft, - &SectorsDone); - } + MaxSegments = UseIndirect(Pdo, SectorsLeft); + + Status = PrepareBlkifReadWrite(Pdo, + Request, + &SGList, + MaxSegments, + SectorStart, + SectorsLeft, + &SectorsDone); if (!NT_SUCCESS(Status)) goto fail; + if (MaxSegments > BLKIF_MAX_SEGMENTS_PER_REQUEST) { + Status = PrepareBlkifIndirect(Pdo, + Request); + if (!NT_SUCCESS(Status)) + goto fail; + } + SectorsLeft -= SectorsDone; SectorStart += SectorsDone; } @@ -1473,11 +1385,15 @@ PrepareReadWrite( // completed preparing SRB, move requests to pending queue for (;;) { PXENVBD_REQUEST Request; - PLIST_ENTRY Entry = RemoveHeadList(&ReqList); + PLIST_ENTRY Entry; + + Entry = RemoveHeadList(&ReqList); if (Entry == &ReqList) break; + ++SrbExt->Count; Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry); + __PdoIncBlkifOpCount(Pdo, Request); QueueAppend(&Pdo->PreparedReqs, Entry); } @@ -1486,14 +1402,16 @@ PrepareReadWrite( fail: for (;;) { PXENVBD_REQUEST Request; - PLIST_ENTRY Entry = RemoveHeadList(&ReqList); + PLIST_ENTRY Entry; + + Entry = RemoveHeadList(&ReqList); if (Entry == &ReqList) break; Request = CONTAINING_RECORD(Entry, XENVBD_REQUEST, Entry); + RequestCleanup(Pdo, Request); __LookasideFree(&Pdo->RequestList, Request); - InterlockedDecrement(&SrbExt->Count); } ASSERT3S(SrbExt->Count, ==, 0); ASSERT(!NT_SUCCESS(Status)); @@ -1519,8 +1437,8 @@ PrepareSyncCache( Request->Srb = Srb; Request->Id = PdoGetTag(Pdo); Request->Operation = BLKIF_OP_WRITE_BARRIER; - - Request->u.Barrier.FirstSector = Cdb_LogicalBlock(Srb); + Request->FirstSector = Cdb_LogicalBlock(Srb); + InitializeListHead(&Request->Segments); __PdoIncBlkifOpCount(Pdo, Request); QueueAppend(&Pdo->PreparedReqs, &Request->Entry); @@ -1546,10 +1464,10 @@ PrepareUnmap( Request->Srb = Srb; Request->Id = PdoGetTag(Pdo); Request->Operation = BLKIF_OP_DISCARD; - - Request->u.Discard.FirstSector = Cdb_LogicalBlock(Srb); - Request->u.Discard.NrSectors = Cdb_TransferBlock(Srb); - Request->u.Discard.Flags = 0; + Request->FirstSector = Cdb_LogicalBlock(Srb); + Request->NrSectors = Cdb_TransferBlock(Srb); + Request->Flags = 0; + InitializeListHead(&Request->Segments); __PdoIncBlkifOpCount(Pdo, Request); QueueAppend(&Pdo->PreparedReqs, &Request->Entry); diff --git a/src/xenvbd/srbext.h b/src/xenvbd/srbext.h index bbb965d..6550b72 100644 --- a/src/xenvbd/srbext.h +++ b/src/xenvbd/srbext.h @@ -37,61 +37,47 @@ #include #include "assert.h" -// Segments - extension of blkif_segment_t -typedef struct _XENVBD_SEGMENT { - PVOID Grant; - UCHAR FirstSector; - UCHAR LastSector; -} XENVBD_SEGMENT, *PXENVBD_SEGMENT; - -typedef struct _XENVBD_MAPPING { - PVOID BufferId; - PVOID Buffer; // VirtAddr mapped to PhysAddr(s) - ULONG Length; - MDL Mdl; - PFN_NUMBER Pfn[2]; -} XENVBD_MAPPING, *PXENVBD_MAPPING; +#pragma pack(push, 1) +typedef struct _BLKIF_SEGMENT { + ULONG GrantRef; + UCHAR First; + UCHAR Last; + USHORT __Padding; +} BLKIF_SEGMENT, *PBLKIF_SEGMENT; +#pragma pack(pop) -// Request - extension of blkif_request_t -typedef struct _XENVBD_REQUEST_READWRITE { - UCHAR NrSegments; - ULONG64 FirstSector; - XENVBD_SEGMENT Segments[BLKIF_MAX_SEGMENTS_PER_REQUEST]; - XENVBD_MAPPING Mappings[BLKIF_MAX_SEGMENTS_PER_REQUEST]; -} XENVBD_REQUEST_READWRITE, *PXENVBD_REQUEST_READWRITE; +#define XENVBD_MAX_SEGMENTS_PER_PAGE (PAGE_SIZE / sizeof(BLKIF_SEGMENT)) -typedef struct _XENVBD_REQUEST_BARRIER { - ULONG64 FirstSector; -} XENVBD_REQUEST_BARRIER, *PXENVBD_REQUEST_BARRIER; +// Internal segment context +typedef struct _XENVBD_SEGMENT { + LIST_ENTRY Entry; + PVOID Grant; + UCHAR FirstSector; + UCHAR LastSector; + ULONG Length; + PVOID BufferId; + PVOID Buffer; // VirtAddr mapped to PhysAddr(s) + MDL Mdl; + PFN_NUMBER Pfn[2]; +} XENVBD_SEGMENT, *PXENVBD_SEGMENT; -typedef struct _XENVBD_REQUEST_DISCARD { - UCHAR Flags; // {0, BLKIF_DISCARD_SECURE} - ULONG64 FirstSector; - ULONG64 NrSectors; -} XENVBD_REQUEST_DISCARD, *PXENVBD_REQUEST_DISCARD; +// Internal request context +typedef struct _XENVBD_REQUEST { + PSCSI_REQUEST_BLOCK Srb; + LIST_ENTRY Entry; + ULONG Id; -typedef struct _XENVBD_REQUEST_INDIRECT { - UCHAR Operation; // BLKIF_OP_{READ/WRITE} - USHORT NrSegments; // 1-4096 - ULONG64 FirstSector; - PVOID Grants[BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST]; - PXENVBD_SEGMENT Segments[BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST]; - PXENVBD_MAPPING Mappings[BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST]; -} XENVBD_REQUEST_INDIRECT, *PXENVBD_REQUEST_INDIRECT; + UCHAR Operation; // BLKIF_OP_{READ/WRITE/BARRIER/DISCARD} + UCHAR Flags; // BLKIF_OP_DISCARD only + USHORT NrSegments; // BLKIF_OP_{READ/WRITE} only, 0-11 (direct) or 11-4096 (indirect) + LIST_ENTRY Segments; // BLKIF_OP_{READ/WRITE} only -typedef struct _XENVBD_REQUEST { - PSCSI_REQUEST_BLOCK Srb; - LIST_ENTRY Entry; - ULONG Id; // atomically increasing number + ULONG64 FirstSector; + ULONG64 NrSectors; // BLKIF_OP_DISCARD only - UCHAR Operation; - union _XENVBD_REQUEST_TYPE { - XENVBD_REQUEST_READWRITE ReadWrite; // BLKIF_OP_{READ/WRITE} - XENVBD_REQUEST_BARRIER Barrier; // BLKIF_OP_WRITE_BARRIER - // nothing // BLKIF_OP_FLUSH_DISKCACHE - XENVBD_REQUEST_DISCARD Discard; // BLKIF_OP_DISCARD - XENVBD_REQUEST_INDIRECT Indirect; // BLKIF_OP_INDIRECT - } u; + // BLKIF_OP_{READ/WRITE} with NrSegments > 11 only + PVOID Pages[BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST]; + PVOID Grants[BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST]; } XENVBD_REQUEST, *PXENVBD_REQUEST; // SRBExtension - context for SRBs