The new version of the interface no longer exposes raw references to clients.
They are now behind an opaque XENVIF_GNTTAB_DESCRIPTOR, analogous to the
XENVIF_EVTCHN_DESCRIPTOR which hides raw ports from clients.
The allocator is also more sophisticated. It uses a pool with per-cpu
magazines for lockless access backed by rangesets.
Signed-off-by: Paul Durrant <paul.durrant@citrix.com>
GNTTAB_ENTRY_FULL_PAGE
} XENBUS_GNTTAB_ENTRY_TYPE, *PXENBUS_GNTTAB_ENTRY_TYPE;
-typedef struct _XENBUS_GNTTAB_COPY_OPERATION {
- LIST_ENTRY ListEntry;
- USHORT RemoteDomain;
- ULONG RemoteReference;
- ULONG RemoteOffset;
- PFN_NUMBER Pfn;
- ULONG Offset;
- ULONG Length;
-} XENBUS_GNTTAB_COPY_OPERATION, *PXENBUS_GNTTAB_COPY_OPERATION;
+typedef struct _XENBUS_GNTTAB_DESCRIPTOR XENBUS_GNTTAB_DESCRIPTOR, *PXENBUS_GNTTAB_DESCRIPTOR;
#define DEFINE_GNTTAB_OPERATIONS \
GNTTAB_OPERATION(VOID, \
Acquire, \
( \
- IN PXENBUS_GNTTAB_CONTEXT Context \
+ IN PXENBUS_GNTTAB_CONTEXT Context \
) \
) \
GNTTAB_OPERATION(VOID, \
Release, \
( \
- IN PXENBUS_GNTTAB_CONTEXT Context \
+ IN PXENBUS_GNTTAB_CONTEXT Context \
) \
) \
- GNTTAB_OPERATION(NTSTATUS, \
+ GNTTAB_OPERATION(PXENBUS_GNTTAB_DESCRIPTOR, \
Get, \
( \
- IN PXENBUS_GNTTAB_CONTEXT Context, \
- OUT PULONG Reference \
+ IN PXENBUS_GNTTAB_CONTEXT Context \
) \
) \
GNTTAB_OPERATION(VOID, \
Put, \
( \
- IN PXENBUS_GNTTAB_CONTEXT Context, \
- IN ULONG Reference \
+ IN PXENBUS_GNTTAB_CONTEXT Context, \
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor \
) \
) \
GNTTAB_OPERATION(NTSTATUS, \
PermitForeignAccess, \
( \
- IN PXENBUS_GNTTAB_CONTEXT Context, \
- IN ULONG Reference, \
- IN USHORT Domain, \
- IN XENBUS_GNTTAB_ENTRY_TYPE Type, \
+ IN PXENBUS_GNTTAB_CONTEXT Context, \
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor, \
+ IN USHORT Domain, \
+ IN XENBUS_GNTTAB_ENTRY_TYPE Type, \
... \
) \
) \
GNTTAB_OPERATION(NTSTATUS, \
RevokeForeignAccess, \
( \
- IN PXENBUS_GNTTAB_CONTEXT Context, \
- IN ULONG Reference \
+ IN PXENBUS_GNTTAB_CONTEXT Context, \
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor \
) \
) \
- GNTTAB_OPERATION(NTSTATUS, \
- Copy, \
+ GNTTAB_OPERATION(ULONG, \
+ Reference, \
( \
- IN PXENBUS_GNTTAB_CONTEXT Context, \
- IN PLIST_ENTRY List, \
- IN ULONG Count \
+ IN PXENBUS_GNTTAB_CONTEXT Context, \
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor \
) \
)
0xd6,
0xe);
-#define GNTTAB_INTERFACE_VERSION 3
+#define GNTTAB_INTERFACE_VERSION 4
#define GNTTAB_OPERATIONS(_Interface) \
(PXENBUS_GNTTAB_OPERATIONS *)((ULONG_PTR)(_Interface))
<ClCompile Include="..\..\src\xenbus\thread.c" />
<ClCompile Include="..\..\src\xenbus\range_set.c" />
<ClCompile Include="..\..\src\xenbus\balloon.c" />
+ <ClCompile Include="..\..\src\xenbus\pool.c" />
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="..\..\src\xenbus\xenbus.rc" />
#include "gnttab.h"
#include "fdo.h"
+#include "range_set.h"
+#include "pool.h"
#include "dbg_print.h"
#include "assert.h"
-#define GNTTAB_MAXIMUM_FRAME_COUNT 32
-#define GNTTAB_ENTRY_PER_FRAME (PAGE_SIZE / sizeof (grant_entry_v1_t))
+#define GNTTAB_MAXIMUM_FRAME_COUNT 32
+#define GNTTAB_ENTRY_PER_FRAME (PAGE_SIZE / sizeof (grant_entry_v1_t))
// Xen requires that we avoid the first 8 entries of the table and
// we also reserve 1 entry for the crash kernel
#define GNTTAB_RESERVED_ENTRY_COUNT 9
-#define GNTTAB_INVALID_REFERENCE 0
-
-#define GNTTAB_IS_INVALID_REFERENCE(_Reference) \
- ((_Reference) < GNTTAB_RESERVED_ENTRY_COUNT)
-
-typedef struct _GNTTAB_DESCRIPTOR {
- LIST_ENTRY ListEntry;
- ULONG Next;
- PVOID Caller;
+struct _XENBUS_GNTTAB_DESCRIPTOR {
+ ULONG Reference;
grant_entry_v1_t Entry;
-} GNTTAB_DESCRIPTOR, *PGNTTAB_DESCRIPTOR;
+};
struct _XENBUS_GNTTAB_CONTEXT {
LONG References;
ULONG FrameCount;
grant_entry_v1_t *Entry;
KSPIN_LOCK Lock;
- ULONG HeadFreeReference;
- PULONG TailFreeReference;
- GNTTAB_DESCRIPTOR Descriptor[GNTTAB_MAXIMUM_FRAME_COUNT * GNTTAB_ENTRY_PER_FRAME];
- LIST_ENTRY List;
+ PXENBUS_RANGE_SET RangeSet;
+ PXENBUS_POOL DescriptorPool;
PXENBUS_SUSPEND_INTERFACE SuspendInterface;
PXENBUS_SUSPEND_CALLBACK SuspendCallbackEarly;
PXENBUS_DEBUG_INTERFACE DebugInterface;
__FreePoolWithTag(Buffer, GNTTAB_TAG);
}
-static NTSTATUS
-GnttabExpand(
+static FORCEINLINE NTSTATUS
+__GnttabExpand(
IN PXENBUS_GNTTAB_CONTEXT Context
)
{
ULONG FrameIndex;
PFN_NUMBER Pfn;
- ULONG Start;
- ULONG End;
- ULONG Reference;
+ ULONGLONG Start;
+ ULONGLONG End;
NTSTATUS status;
FrameIndex = Context->FrameCount;
Start = __max(GNTTAB_RESERVED_ENTRY_COUNT, FrameIndex * GNTTAB_ENTRY_PER_FRAME);
End = (Context->FrameCount * GNTTAB_ENTRY_PER_FRAME) - 1;
- Trace("adding refrences [%08x - %08x]\n", Start, End);
+ Trace("adding refrences [%08llx - %08llx]\n", Start, End);
- for (Reference = Start; Reference <= End; Reference++) {
- PGNTTAB_DESCRIPTOR Descriptor = &Context->Descriptor[Reference];
-
- *Context->TailFreeReference = Reference;
-
- ASSERT(GNTTAB_IS_INVALID_REFERENCE(Descriptor->Next));
- Context->TailFreeReference = &Descriptor->Next;
- }
+ RangeSetPut(Context->RangeSet, Start, End);
return STATUS_SUCCESS;
return status;
}
-static FORCEINLINE VOID
-__GnttabEmpty(
- IN PXENBUS_GNTTAB_CONTEXT Context
+static NTSTATUS
+GnttabDescriptorCtor(
+ IN PVOID Argument,
+ IN PVOID Object
)
{
- while (!GNTTAB_IS_INVALID_REFERENCE(Context->HeadFreeReference)) {
- ULONG Reference = Context->HeadFreeReference;
- PGNTTAB_DESCRIPTOR Descriptor = &Context->Descriptor[Reference];
+ PXENBUS_GNTTAB_CONTEXT Context = Argument;
+ PXENBUS_GNTTAB_DESCRIPTOR Descriptor = Object;
+ NTSTATUS status;
- Context->HeadFreeReference = Descriptor->Next;
- Descriptor->Next = GNTTAB_INVALID_REFERENCE;
- }
+ if (!RangeSetIsEmpty(Context->RangeSet))
+ goto done;
- Context->TailFreeReference = &Context->HeadFreeReference;
-}
+ status = __GnttabExpand(Context);
+ if (!NT_SUCCESS(status))
+ goto fail1;
-extern USHORT
-RtlCaptureStackBackTrace(
- __in ULONG FramesToSkip,
- __in ULONG FramesToCapture,
- __out PVOID *BackTrace,
- __out_opt PULONG BackTraceHash
- );
+done:
+ Descriptor->Reference = (ULONG)RangeSetPop(Context->RangeSet);
-static NTSTATUS
-GnttabGet(
- IN PXENBUS_GNTTAB_CONTEXT Context,
- OUT PULONG Reference
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static VOID
+GnttabDescriptorDtor(
+ IN PVOID Argument,
+ IN PVOID Object
)
{
- PGNTTAB_DESCRIPTOR Descriptor;
- KIRQL Irql;
+ PXENBUS_GNTTAB_CONTEXT Context = Argument;
+ PXENBUS_GNTTAB_DESCRIPTOR Descriptor = Object;
NTSTATUS status;
- KeAcquireSpinLock(&Context->Lock, &Irql);
-
- if (!GNTTAB_IS_INVALID_REFERENCE(Context->HeadFreeReference))
- goto done;
+ status = RangeSetPut(Context->RangeSet,
+ (ULONGLONG)Descriptor->Reference,
+ (ULONGLONG)Descriptor->Reference);
+ ASSERT(NT_SUCCESS(status));
+}
- ASSERT3P(Context->TailFreeReference, ==, &Context->HeadFreeReference);
+static FORCEINLINE VOID
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__GnttabAcquireLock(
+ IN PXENBUS_GNTTAB_CONTEXT Context
+ )
+{
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
- status = GnttabExpand(Context);
- if (!NT_SUCCESS(status))
- goto fail1;
+ KeAcquireSpinLockAtDpcLevel(&Context->Lock);
+}
-done:
- ASSERT(!GNTTAB_IS_INVALID_REFERENCE(Context->HeadFreeReference));
+static DECLSPEC_NOINLINE VOID
+GnttabAcquireLock(
+ IN PXENBUS_GNTTAB_CONTEXT Context
+ )
+{
+ __GnttabAcquireLock(Context);
+}
- *Reference = Context->HeadFreeReference;
- Descriptor = &Context->Descriptor[*Reference];
+static FORCEINLINE VOID
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__GnttabReleaseLock(
+ IN PXENBUS_GNTTAB_CONTEXT Context
+ )
+{
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
- Context->HeadFreeReference = Descriptor->Next;
+#pragma prefast(disable:26110)
+ KeReleaseSpinLockFromDpcLevel(&Context->Lock);
+}
- if (GNTTAB_IS_INVALID_REFERENCE(Context->HeadFreeReference))
- Context->TailFreeReference = &Context->HeadFreeReference;
+static DECLSPEC_NOINLINE VOID
+GnttabReleaseLock(
+ IN PXENBUS_GNTTAB_CONTEXT Context
+ )
+{
+ __GnttabReleaseLock(Context);
+}
- Descriptor->Next = GNTTAB_INVALID_REFERENCE;
+static FORCEINLINE NTSTATUS
+__GnttabFill(
+ IN PXENBUS_GNTTAB_CONTEXT Context
+ )
+{
+ NTSTATUS status;
- ASSERT(IsZeroMemory(Descriptor, sizeof (GNTTAB_DESCRIPTOR)));
+ status = RangeSetInitialize(&Context->RangeSet);
+ if (!NT_SUCCESS(status))
+ goto fail1;
- InsertTailList(&Context->List, &Descriptor->ListEntry);
+ status = PoolInitialize("GnttabDescriptor",
+ sizeof (XENBUS_GNTTAB_DESCRIPTOR),
+ GnttabDescriptorCtor,
+ GnttabDescriptorDtor,
+ GnttabAcquireLock,
+ GnttabReleaseLock,
+ Context,
+ &Context->DescriptorPool);
+ if (!NT_SUCCESS(status))
+ goto fail2;
- KeReleaseSpinLock(&Context->Lock, Irql);
+ return STATUS_SUCCESS;
- (VOID) RtlCaptureStackBackTrace(1, 1, &Descriptor->Caller, NULL);
+fail2:
+ Error("fail2\n");
- return STATUS_SUCCESS;
+ RangeSetTeardown(Context->RangeSet);
+ Context->RangeSet = NULL;
fail1:
Error("fail1 (%08x)\n", status);
- KeReleaseSpinLock(&Context->Lock, Irql);
-
return status;
}
-static VOID
-GnttabPut(
- IN PXENBUS_GNTTAB_CONTEXT Context,
- IN ULONG Reference
+static FORCEINLINE VOID
+__GnttabEmpty(
+ IN PXENBUS_GNTTAB_CONTEXT Context
)
{
- KIRQL Irql;
- PGNTTAB_DESCRIPTOR Descriptor;
+ ULONGLONG Entry;
- ASSERT(!GNTTAB_IS_INVALID_REFERENCE(Reference));
+ PoolTeardown(Context->DescriptorPool);
+ Context->DescriptorPool = NULL;
- Descriptor = &Context->Descriptor[Reference];
+ for (Entry = GNTTAB_RESERVED_ENTRY_COUNT;
+ Entry < Context->FrameCount * GNTTAB_ENTRY_PER_FRAME;
+ Entry++) {
+ NTSTATUS status;
- Descriptor->Caller = NULL;
-
- KeAcquireSpinLock(&Context->Lock, &Irql);
-
- RemoveEntryList(&Descriptor->ListEntry);
- RtlZeroMemory(&Descriptor->ListEntry, sizeof (LIST_ENTRY));
-
- ASSERT(IsZeroMemory(Descriptor, sizeof (GNTTAB_DESCRIPTOR)));
-
- ASSERT(GNTTAB_IS_INVALID_REFERENCE(Descriptor->Next));
+ status = RangeSetGet(Context->RangeSet, Entry);
+ ASSERT(NT_SUCCESS(status));
+ }
- *Context->TailFreeReference = Reference;
+ RangeSetTeardown(Context->RangeSet);
+ Context->RangeSet = NULL;
+}
- ASSERT(GNTTAB_IS_INVALID_REFERENCE(Descriptor->Next));
- Context->TailFreeReference = &Descriptor->Next;
+static PXENBUS_GNTTAB_DESCRIPTOR
+GnttabGet(
+ IN PXENBUS_GNTTAB_CONTEXT Context
+ )
+{
+ return PoolGet(Context->DescriptorPool, FALSE);
+}
- KeReleaseSpinLock(&Context->Lock, Irql);
+static VOID
+GnttabPut(
+ IN PXENBUS_GNTTAB_CONTEXT Context,
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor
+ )
+{
+ PoolPut(Context->DescriptorPool, Descriptor, FALSE);
}
static FORCEINLINE NTSTATUS
__GnttabPermitForeignAccessFullPage(
- IN PXENBUS_GNTTAB_CONTEXT Context,
- IN ULONG Reference,
- IN USHORT Domain,
- IN va_list Arguments
+ IN PXENBUS_GNTTAB_CONTEXT Context,
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor,
+ IN USHORT Domain,
+ IN va_list Arguments
)
{
- PGNTTAB_DESCRIPTOR Descriptor;
- PFN_NUMBER Frame;
- BOOLEAN ReadOnly;
- grant_entry_v1_t *Entry;
-
- ASSERT(!GNTTAB_IS_INVALID_REFERENCE(Reference));
+ PFN_NUMBER Frame;
+ BOOLEAN ReadOnly;
+ grant_entry_v1_t *Entry;
Frame = va_arg(Arguments, PFN_NUMBER);
ReadOnly = va_arg(Arguments, BOOLEAN);
- Descriptor = &Context->Descriptor[Reference];
ASSERT(IsZeroMemory(&Descriptor->Entry, sizeof (grant_entry_v1_t)));
Descriptor->Entry.flags = (ReadOnly) ? GTF_readonly : 0;
Descriptor->Entry.frame = (uint32_t)Frame;
ASSERT3U(Descriptor->Entry.frame, ==, Frame);
- Entry = &Context->Entry[Reference];
+ Entry = &Context->Entry[Descriptor->Reference];
*Entry = Descriptor->Entry;
KeMemoryBarrier();
static NTSTATUS
GnttabPermitForeignAccess(
IN PXENBUS_GNTTAB_CONTEXT Context,
- IN ULONG Reference,
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor,
IN USHORT Domain,
IN XENBUS_GNTTAB_ENTRY_TYPE Type,
...
va_start(Arguments, Type);
switch (Type) {
case GNTTAB_ENTRY_FULL_PAGE:
- status =__GnttabPermitForeignAccessFullPage(Context, Reference, Domain, Arguments);
+ status =__GnttabPermitForeignAccessFullPage(Context, Descriptor, Domain, Arguments);
break;
default:
static NTSTATUS
GnttabRevokeForeignAccess(
- IN PXENBUS_GNTTAB_CONTEXT Context,
- IN ULONG Reference
+ IN PXENBUS_GNTTAB_CONTEXT Context,
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor
)
{
- grant_entry_v1_t *Entry;
- volatile SHORT *Flags;
- PGNTTAB_DESCRIPTOR Descriptor;
- ULONG Attempt;
- NTSTATUS status;
+ grant_entry_v1_t *Entry;
+ volatile SHORT *Flags;
+ ULONG Attempt;
+ NTSTATUS status;
- Entry = &Context->Entry[Reference];
+ Entry = &Context->Entry[Descriptor->Reference];
Flags = (volatile SHORT *)&Entry->flags;
Attempt = 0;
goto fail1;
RtlZeroMemory(Entry, sizeof (grant_entry_v1_t));
-
- Descriptor = &Context->Descriptor[Reference];
RtlZeroMemory(&Descriptor->Entry, sizeof (grant_entry_v1_t));
return STATUS_SUCCESS;
return status;
}
-static NTSTATUS
-GnttabCopy(
- IN PXENBUS_GNTTAB_CONTEXT Context,
- IN PLIST_ENTRY List,
- IN ULONG Count
+static ULONG
+GnttabReference(
+ IN PXENBUS_GNTTAB_CONTEXT Context,
+ IN PXENBUS_GNTTAB_DESCRIPTOR Descriptor
)
{
- struct gnttab_copy *op;
- PLIST_ENTRY ListEntry;
- ULONG Index;
- NTSTATUS status;
-
UNREFERENCED_PARAMETER(Context);
- op = __GnttabAllocate(sizeof (struct gnttab_copy) * Count);
-
- status = STATUS_NO_MEMORY;
- if (op == NULL)
- goto fail1;
-
- Index = 0;
- for (ListEntry = List->Flink;
- ListEntry != List;
- ListEntry = ListEntry->Flink) {
- PXENBUS_GNTTAB_COPY_OPERATION Operation;
-
- Operation = CONTAINING_RECORD(ListEntry, XENBUS_GNTTAB_COPY_OPERATION, ListEntry);
-
- ASSERT3U(Operation->Length, <=, PAGE_SIZE);
- ASSERT3U(Operation->Offset + Operation->Length, <=, PAGE_SIZE);
- ASSERT3U(Operation->RemoteOffset + Operation->Length, <=, PAGE_SIZE);
-
- op[Index].source.domid = Operation->RemoteDomain;
- op[Index].source.u.ref = Operation->RemoteReference;
- op[Index].source.offset = (USHORT)Operation->RemoteOffset;
-
- op[Index].dest.domid = DOMID_SELF;
- op[Index].dest.u.gmfn = Operation->Pfn;
- op[Index].dest.offset = (USHORT)Operation->Offset;
-
- op[Index].len = (USHORT)Operation->Length;
- op[Index].flags = GNTCOPY_source_gref;
-
- Index++;
- }
- ASSERT3U(Index, ==, Count);
-
- status = GrantTableCopy(op, Count);
- if (!NT_SUCCESS(status))
- goto fail2;
-
- status = STATUS_UNSUCCESSFUL;
- for (Index = 0; Index < Count; Index++) {
- if (op[Index].status != 0)
- goto fail3;
- }
-
- __GnttabFree(op);
-
- return STATUS_SUCCESS;
-
-fail3:
- Error("fail3\n");
-
-fail2:
- Error("fail2\n");
-
- __GnttabFree(op);
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
+ return (ULONG)Descriptor->Reference;
}
static VOID
)
{
PXENBUS_GNTTAB_CONTEXT Context = Argument;
+ ULONG Allocated;
+ ULONG MaximumAllocated;
+ ULONG Count;
+ ULONG MinimumCount;
UNREFERENCED_PARAMETER(Crashing);
Context->DebugCallback,
"FrameCount = %u\n",
Context->FrameCount);
+
+ PoolGetStatistics(Context->DescriptorPool,
+ &Allocated,
+ &MaximumAllocated,
+ &Count,
+ &MinimumCount);
+
+ DEBUG(Printf,
+ Context->DebugInterface,
+ Context->DebugCallback,
+ "DESCRIPTOR POOL: Allocated = %u (Maximum = %u)\n",
+ Allocated,
+ MaximumAllocated);
+
+ DEBUG(Printf,
+ Context->DebugInterface,
+ Context->DebugCallback,
+ "DESCRIPTOR POOL: Count = %u (Minimum = %u)\n",
+ Count,
+ MinimumCount);
}
NTSTATUS
Info("grant_entry_v1_t *: %p\n", Context->Entry);
- InitializeListHead(&Context->List);
KeInitializeSpinLock(&Context->Lock);
- ASSERT(GNTTAB_IS_INVALID_REFERENCE(Context->HeadFreeReference));
- Context->TailFreeReference = &Context->HeadFreeReference;
-
- GnttabExpand(Context);
+ __GnttabFill(Context);
Context->SuspendInterface = FdoGetSuspendInterface(Fdo);
__GnttabEmpty(Context);
- Context->TailFreeReference = NULL;
-
RtlZeroMemory(&Context->Lock, sizeof (KSPIN_LOCK));
- RtlZeroMemory(&Context->List, sizeof (LIST_ENTRY));
Context->Entry = NULL;
Trace("====>\n");
- if (!IsListEmpty(&Context->List))
- BUG("OUTSTANDING REFERENCES");
-
DEBUG(Deregister,
Context->DebugInterface,
Context->DebugCallback);
__GnttabEmpty(Context);
- Context->TailFreeReference = NULL;
-
RtlZeroMemory(&Context->Lock, sizeof (KSPIN_LOCK));
- RtlZeroMemory(&Context->List, sizeof (LIST_ENTRY));
Context->Entry = NULL;
--- /dev/null
+/* Copyright (c) Citrix Systems Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms,
+ * with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer in the documentation and/or other
+ * materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <ntddk.h>
+#include <ntstrsafe.h>
+#include <util.h>
+
+#include "pool.h"
+#include "dbg_print.h"
+#include "assert.h"
+
+#define POOL_POOL 'OBJE'
+
+typedef struct _OBJECT_HEADER {
+ ULONG Magic;
+
+#define OBJECT_HEADER_MAGIC 0x02121996
+
+ LIST_ENTRY ListEntry;
+} OBJECT_HEADER, *POBJECT_HEADER;
+
+#define MAXIMUM_SLOTS 6
+
+typedef struct _POOL_MAGAZINE {
+ PVOID Slot[MAXIMUM_SLOTS];
+} POOL_MAGAZINE, *PPOOL_MAGAZINE;
+
+struct _XENBUS_POOL {
+ const CHAR *Name;
+ ULONG Size;
+ NTSTATUS (*Ctor)(PVOID, PVOID);
+ VOID (*Dtor)(PVOID, PVOID);
+ VOID (*AcquireLock)(PVOID);
+ VOID (*ReleaseLock)(PVOID);
+ PVOID Argument;
+ KTIMER Timer;
+ KDPC Dpc;
+ LIST_ENTRY GetList;
+ PLIST_ENTRY PutList;
+ POOL_MAGAZINE Magazine[MAXIMUM_PROCESSORS];
+ LONG Allocated;
+ LONG MaximumAllocated;
+ LONG Count;
+ LONG MinimumCount;
+};
+
+static FORCEINLINE PVOID
+__PoolAllocate(
+ IN ULONG Length
+ )
+{
+ return __AllocateNonPagedPoolWithTag(Length, POOL_POOL);
+}
+
+static FORCEINLINE VOID
+__PoolFree(
+ IN PVOID Buffer
+ )
+{
+ __FreePoolWithTag(Buffer, POOL_POOL);
+}
+
+static FORCEINLINE VOID
+__PoolSwizzle(
+ IN PXENBUS_POOL Pool
+ )
+{
+ PLIST_ENTRY ListEntry;
+
+ ListEntry = InterlockedExchangePointer(&Pool->PutList, NULL);
+
+ while (ListEntry != NULL) {
+ PLIST_ENTRY Next;
+
+ Next = ListEntry->Flink;
+ ListEntry->Flink = NULL;
+ ASSERT3P(ListEntry->Blink, ==, NULL);
+
+ InsertTailList(&Pool->GetList, ListEntry);
+
+ ListEntry = Next;
+ }
+}
+
+static FORCEINLINE PVOID
+__PoolGetShared(
+ IN PXENBUS_POOL Pool,
+ IN BOOLEAN Locked
+ )
+{
+ LONG Count;
+ POBJECT_HEADER Header;
+ PVOID Object;
+ LONG Allocated;
+ NTSTATUS status;
+
+ Count = InterlockedDecrement(&Pool->Count);
+
+ if (Count >= 0) {
+ PLIST_ENTRY ListEntry;
+
+ if (!Locked)
+ Pool->AcquireLock(Pool->Argument);
+
+ if (Count < Pool->MinimumCount)
+ Pool->MinimumCount = Count;
+
+ if (IsListEmpty(&Pool->GetList))
+ __PoolSwizzle(Pool);
+
+ ListEntry = RemoveHeadList(&Pool->GetList);
+ ASSERT(ListEntry != &Pool->GetList);
+
+ if (!Locked)
+ Pool->ReleaseLock(Pool->Argument);
+
+ RtlZeroMemory(ListEntry, sizeof (LIST_ENTRY));
+
+ Header = CONTAINING_RECORD(ListEntry, OBJECT_HEADER, ListEntry);
+ ASSERT3U(Header->Magic, ==, OBJECT_HEADER_MAGIC);
+
+ Object = Header + 1;
+ goto done;
+ }
+
+ (VOID) InterlockedIncrement(&Pool->Count);
+
+ Header = __PoolAllocate(sizeof (OBJECT_HEADER) + Pool->Size);
+
+ status = STATUS_NO_MEMORY;
+ if (Header == NULL)
+ goto fail1;
+
+ Header->Magic = OBJECT_HEADER_MAGIC;
+
+ Object = Header + 1;
+
+ status = Pool->Ctor(Pool->Argument, Object);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+ Allocated = InterlockedIncrement(&Pool->Allocated);
+
+ if (Allocated > Pool->MaximumAllocated) {
+ if (!Locked)
+ Pool->AcquireLock(Pool->Argument);
+
+ if (Allocated > Pool->MaximumAllocated)
+ Pool->MaximumAllocated = Allocated;
+
+ if (!Locked)
+ Pool->ReleaseLock(Pool->Argument);
+ }
+
+done:
+ return Object;
+
+fail2:
+ Error("fail2\n");
+
+ Header->Magic = 0;
+
+ ASSERT(IsZeroMemory(Header, sizeof (OBJECT_HEADER)));
+ __PoolFree(Header);
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return NULL;
+}
+
+static FORCEINLINE VOID
+__PoolPutShared(
+ IN PXENBUS_POOL Pool,
+ IN PVOID Object,
+ IN BOOLEAN Locked
+ )
+{
+ POBJECT_HEADER Header;
+ PLIST_ENTRY Old;
+ PLIST_ENTRY New;
+
+ ASSERT(Object != NULL);
+
+ Header = Object;
+ --Header;
+ ASSERT3U(Header->Magic, ==, OBJECT_HEADER_MAGIC);
+
+ ASSERT(IsZeroMemory(&Header->ListEntry, sizeof (LIST_ENTRY)));
+
+ if (!Locked) {
+ New = &Header->ListEntry;
+
+ do {
+ Old = Pool->PutList;
+ New->Flink = Old;
+ } while (InterlockedCompareExchangePointer(&Pool->PutList, New, Old) != Old);
+ } else {
+ InsertTailList(&Pool->GetList, &Header->ListEntry);
+ }
+
+ KeMemoryBarrier();
+
+ (VOID) InterlockedIncrement(&Pool->Count);
+}
+
+static FORCEINLINE PVOID
+__PoolGetMagazine(
+ IN PXENBUS_POOL Pool,
+ IN ULONG Cpu
+ )
+{
+ PPOOL_MAGAZINE Magazine;
+ ULONG Index;
+
+ Magazine = &Pool->Magazine[Cpu];
+
+ for (Index = 0; Index < MAXIMUM_SLOTS; Index++) {
+ PVOID Object;
+
+ if (Magazine->Slot[Index] != NULL) {
+ Object = Magazine->Slot[Index];
+ Magazine->Slot[Index] = NULL;
+
+ return Object;
+ }
+ }
+
+ return NULL;
+}
+
+static FORCEINLINE BOOLEAN
+__PoolPutMagazine(
+ IN PXENBUS_POOL Pool,
+ IN ULONG Cpu,
+ IN PVOID Object
+ )
+{
+ PPOOL_MAGAZINE Magazine;
+ ULONG Index;
+
+ Magazine = &Pool->Magazine[Cpu];
+
+ for (Index = 0; Index < MAXIMUM_SLOTS; Index++) {
+ if (Magazine->Slot[Index] == NULL) {
+ Magazine->Slot[Index] = Object;
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+PVOID
+PoolGet(
+ IN PXENBUS_POOL Pool,
+ IN BOOLEAN Locked
+ )
+{
+ KIRQL Irql;
+ ULONG Cpu;
+ PVOID Object;
+
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ Cpu = KeGetCurrentProcessorNumber();
+
+ Object = __PoolGetMagazine(Pool, Cpu);
+ if (Object == NULL)
+ Object = __PoolGetShared(Pool, Locked);
+
+ KeLowerIrql(Irql);
+
+ return Object;
+}
+
+VOID
+PoolPut(
+ IN PXENBUS_POOL Pool,
+ IN PVOID Object,
+ IN BOOLEAN Locked
+ )
+{
+ KIRQL Irql;
+ ULONG Cpu;
+
+ KeRaiseIrql(DISPATCH_LEVEL, &Irql);
+ Cpu = KeGetCurrentProcessorNumber();
+
+ if (!__PoolPutMagazine(Pool, Cpu, Object))
+ __PoolPutShared(Pool, Object, Locked);
+
+ KeLowerIrql(Irql);
+}
+
+VOID
+PoolGetStatistics(
+ IN PXENBUS_POOL Pool,
+ OUT PULONG Allocated,
+ OUT PULONG MaximumAllocated,
+ OUT PULONG Count,
+ OUT PULONG MinimumCount
+ )
+{
+ *Allocated = Pool->Allocated;
+ *MaximumAllocated = Pool->MaximumAllocated;
+
+ *Count = Pool->Count;
+ *MinimumCount = Pool->MinimumCount;
+}
+
+static FORCEINLINE
+__PoolFlushMagazines(
+ IN PXENBUS_POOL Pool
+ )
+{
+ ULONG Cpu;
+
+ for (Cpu = 0; Cpu < MAXIMUM_PROCESSORS; Cpu++) {
+ PVOID Object;
+
+ while ((Object = __PoolGetMagazine(Pool, Cpu)) != NULL)
+ __PoolPutShared(Pool, Object, TRUE);
+ }
+}
+
+static FORCEINLINE VOID
+__PoolTrimShared(
+ IN PXENBUS_POOL Pool,
+ IN OUT PLIST_ENTRY List
+ )
+{
+ LONG Count;
+ LONG Excess;
+
+ Count = Pool->Count;
+
+ KeMemoryBarrier();
+
+ Excess = Pool->MinimumCount;
+
+ while (Excess != 0) {
+ PLIST_ENTRY ListEntry;
+
+ Count = InterlockedDecrement(&Pool->Count);
+ if (Count < 0) {
+ Count = InterlockedIncrement(&Pool->Count);
+ break;
+ }
+
+ if (IsListEmpty(&Pool->GetList))
+ __PoolSwizzle(Pool);
+
+ ListEntry = RemoveHeadList(&Pool->GetList);
+ ASSERT(ListEntry != &Pool->GetList);
+
+ InsertTailList(List, ListEntry);
+
+ InterlockedDecrement(&Pool->Allocated);
+ --Excess;
+ }
+
+ Pool->MinimumCount = Count;
+}
+
+static FORCEINLINE VOID
+__PoolEmpty(
+ IN PXENBUS_POOL Pool,
+ IN OUT PLIST_ENTRY List
+ )
+{
+ while (!IsListEmpty(List)) {
+ PLIST_ENTRY ListEntry;
+ POBJECT_HEADER Header;
+ PVOID Object;
+
+ ListEntry = RemoveHeadList(List);
+ RtlZeroMemory(ListEntry, sizeof (LIST_ENTRY));
+
+ Header = CONTAINING_RECORD(ListEntry, OBJECT_HEADER, ListEntry);
+ ASSERT3U(Header->Magic, ==, OBJECT_HEADER_MAGIC);
+
+ Object = Header + 1;
+
+ Pool->Dtor(Pool->Argument, Object);
+
+ Header->Magic = 0;
+
+ ASSERT(IsZeroMemory(Header, sizeof (OBJECT_HEADER)));
+ __PoolFree(Header);
+ }
+}
+
+#define TIME_US(_us) ((_us) * 10)
+#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
+#define TIME_RELATIVE(_t) (-(_t))
+
+#define POOL_PERIOD 1000
+
+KDEFERRED_ROUTINE PoolDpc;
+
+VOID
+PoolDpc(
+ IN PKDPC Dpc,
+ IN PVOID Context,
+ IN PVOID Argument1,
+ IN PVOID Argument2
+ )
+{
+ PXENBUS_POOL Pool = Context;
+ LIST_ENTRY List;
+
+ UNREFERENCED_PARAMETER(Dpc);
+ UNREFERENCED_PARAMETER(Argument1);
+ UNREFERENCED_PARAMETER(Argument2);
+
+ InitializeListHead(&List);
+
+ Pool->AcquireLock(Pool->Argument);
+ __PoolTrimShared(Pool, &List);
+ Pool->ReleaseLock(Pool->Argument);
+
+ __PoolEmpty(Pool, &List);
+ ASSERT(IsListEmpty(&List));
+}
+
+NTSTATUS
+PoolInitialize(
+ IN const CHAR *Name,
+ IN ULONG Size,
+ IN NTSTATUS (*Ctor)(PVOID, PVOID),
+ IN VOID (*Dtor)(PVOID, PVOID),
+ IN VOID (*AcquireLock)(PVOID),
+ IN VOID (*ReleaseLock)(PVOID),
+ IN PVOID Argument,
+ OUT PXENBUS_POOL *Pool
+ )
+{
+ LARGE_INTEGER Timeout;
+ NTSTATUS status;
+
+ *Pool = __PoolAllocate(sizeof (XENBUS_POOL));
+
+ status = STATUS_NO_MEMORY;
+ if (*Pool == NULL)
+ goto fail1;
+
+ (*Pool)->Name = Name;
+ (*Pool)->Size = Size;
+ (*Pool)->Ctor = Ctor;
+ (*Pool)->Dtor = Dtor;
+ (*Pool)->AcquireLock = AcquireLock;
+ (*Pool)->ReleaseLock = ReleaseLock;
+ (*Pool)->Argument = Argument;
+
+ InitializeListHead(&(*Pool)->GetList);
+
+ KeInitializeDpc(&(*Pool)->Dpc,
+ PoolDpc,
+ (*Pool));
+
+ Timeout.QuadPart = TIME_RELATIVE(TIME_MS(POOL_PERIOD));
+
+ KeInitializeTimer(&(*Pool)->Timer);
+ KeSetTimerEx(&(*Pool)->Timer,
+ Timeout,
+ POOL_PERIOD,
+ &(*Pool)->Dpc);
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+VOID
+PoolTeardown(
+ IN PXENBUS_POOL Pool
+ )
+{
+ LIST_ENTRY List;
+
+ KeCancelTimer(&Pool->Timer);
+ KeFlushQueuedDpcs();
+
+ RtlZeroMemory(&Pool->Timer, sizeof (KTIMER));
+ RtlZeroMemory(&Pool->Dpc, sizeof (KDPC));
+
+ InitializeListHead(&List);
+
+ __PoolFlushMagazines(Pool);
+
+ Pool->MinimumCount = Pool->Count;
+ __PoolTrimShared(Pool, &List);
+ __PoolEmpty(Pool, &List);
+
+ ASSERT3U(Pool->Count, ==, 0);
+ ASSERT3U(Pool->Allocated, ==, 0);
+ Pool->MaximumAllocated = 0;
+
+ RtlZeroMemory(&Pool->GetList, sizeof (LIST_ENTRY));
+
+ Pool->Argument = NULL;
+ Pool->ReleaseLock = NULL;
+ Pool->AcquireLock = NULL;
+ Pool->Dtor = NULL;
+ Pool->Ctor = NULL;
+ Pool->Size = 0;
+ Pool->Name = NULL;
+
+ ASSERT(IsZeroMemory(Pool, sizeof (XENBUS_POOL)));
+ __PoolFree(Pool);
+}
--- /dev/null
+/* Copyright (c) Citrix Systems Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms,
+ * with or without modification, are permitted provided
+ * that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the
+ * following disclaimer in the documentation and/or other
+ * materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _XENBUS_POOL_H
+#define _XENBUS_POOL_H
+
+#include <ntddk.h>
+
+typedef struct _XENBUS_POOL XENBUS_POOL, *PXENBUS_POOL;
+
+extern NTSTATUS
+PoolInitialize(
+ IN const CHAR *Name,
+ IN ULONG Size,
+ IN NTSTATUS (*Ctor)(PVOID, PVOID),
+ IN VOID (*Dtor)(PVOID, PVOID),
+ IN VOID (*AcquireLock)(PVOID),
+ IN VOID (*ReleaseLock)(PVOID),
+ IN PVOID Argument,
+ OUT PXENBUS_POOL *Pool
+ );
+
+extern VOID
+PoolTeardown(
+ IN PXENBUS_POOL Pool
+ );
+
+extern PVOID
+PoolGet(
+ IN PXENBUS_POOL Pool,
+ IN BOOLEAN Locked
+ );
+
+extern VOID
+PoolPut(
+ IN PXENBUS_POOL Pool,
+ IN PVOID Object,
+ IN BOOLEAN Locked
+ );
+
+extern VOID
+PoolGetStatistics(
+ IN PXENBUS_POOL Pool,
+ OUT PULONG Allocated,
+ OUT PULONG MaximumAllocated,
+ OUT PULONG Count,
+ OUT PULONG MinimumCount
+ );
+
+#endif // _XENBUS_POOL_H
__RangeSetRemove(RangeSet, TRUE);
}
+BOOLEAN
+RangeSetIsEmpty(
+ IN PXENBUS_RANGE_SET RangeSet
+ )
+{
+ return IsListEmpty(&RangeSet->List);
+}
+
ULONGLONG
RangeSetPop(
IN PXENBUS_RANGE_SET RangeSet
typedef struct _XENBUS_RANGE_SET XENBUS_RANGE_SET, *PXENBUS_RANGE_SET;
+extern BOOLEAN
+RangeSetIsEmpty(
+ IN PXENBUS_RANGE_SET RangeSet
+ );
+
extern ULONGLONG
RangeSetPop(
IN PXENBUS_RANGE_SET RangeSet
projects / people / pauldu / xenbus.git / commitdiff