...into the new Poller sub-system.
For efficiency it is desirable to have a single DPC handle both Receiver
and Transmitter polling, even if there are separate event channels for
the shared rings.
This patch moves all the basic event and DPC code into the Poller
subsystem, which calls back into the Transmitter and Receiver sub-systems
(to poll the shared rings) from its single per-CPU DPC.
Signed-off-by: Paul Durrant <paul.durrant@citrix.com>
USHORT BackendDomain;
ULONG MaxQueues;
ULONG NumQueues;
- BOOLEAN Split;
ULONG DisableToeplitz;
PXENVIF_MAC Mac;
return __FrontendGetNumQueues(Frontend);
}
-static VOID
-FrontendSetSplit(
- IN PXENVIF_FRONTEND Frontend
- )
-{
- PCHAR Buffer;
- NTSTATUS status;
-
- status = XENBUS_STORE(Read,
- &Frontend->StoreInterface,
- NULL,
- __FrontendGetBackendPath(Frontend),
- "feature-split-event-channels",
- &Buffer);
- if (NT_SUCCESS(status)) {
- Frontend->Split = (BOOLEAN)strtol(Buffer, NULL, 2);
-
- XENBUS_STORE(Free,
- &Frontend->StoreInterface,
- Buffer);
- } else {
- Frontend->Split = FALSE;
- }
-
- Info("%s: %s\n", __FrontendGetPath(Frontend),
- (Frontend->Split) ? "TRUE" : "FALSE");
-}
-
-static FORCEINLINE BOOLEAN
-__FrontendIsSplit(
- IN PXENVIF_FRONTEND Frontend
- )
-{
- return Frontend->Split;
-}
-
-BOOLEAN
-FrontendIsSplit(
- IN PXENVIF_FRONTEND Frontend
- )
-{
- return __FrontendIsSplit(Frontend);
-}
-
static FORCEINLINE NTSTATUS
__FrontendUpdateHash(
PXENVIF_FRONTEND Frontend,
goto fail3;
FrontendSetNumQueues(Frontend);
- FrontendSetSplit(Frontend);
status = PollerConnect(__FrontendGetPoller(Frontend));
if (!NT_SUCCESS(status))
MacDisconnect(__FrontendGetMac(Frontend));
- Frontend->Split = FALSE;
Frontend->NumQueues = 0;
fail3:
PollerDisconnect(__FrontendGetPoller(Frontend));
MacDisconnect(__FrontendGetMac(Frontend));
- Frontend->Split = FALSE;
Frontend->NumQueues = 0;
XENBUS_DEBUG(Deregister,
IN PXENVIF_FRONTEND Frontend
);
-extern BOOLEAN
-FrontendIsSplit(
- IN PXENVIF_FRONTEND Frontend
- );
-
-extern BOOLEAN
-FrontendIsSplit(
- IN PXENVIF_FRONTEND Frontend
- );
-
extern PCHAR
FrontendFormatPath(
IN PXENVIF_FRONTEND Frontend,
#include "pdo.h"
#include "frontend.h"
+#include "transmitter.h"
+#include "receiver.h"
#include "poller.h"
#include "vif.h"
#include "thread.h"
#define MAXNAMELEN 128
+typedef struct _XENVIF_POLLER_INSTANCE XENVIF_POLLER_INSTANCE, *PXENVIF_POLLER_INSTANCE;
+
+typedef enum _XENVIF_POLLER_CHANNEL_TYPE {
+ XENVIF_POLLER_CHANNEL_RECEIVER,
+ XENVIF_POLLER_CHANNEL_TRANSMITTER,
+ XENVIF_POLLER_CHANNEL_COMBINED,
+ XENVIF_POLLER_CHANNEL_TYPE_COUNT
+} XENVIF_POLLER_CHANNEL_TYPE, *PXENVIF_POLLER_CHANNEL_TYPE;
+
+#define XENVIF_POLLER_CHANNEL_INVALID XENVIF_POLLER_CHANNEL_TYPE_COUNT
+
+typedef struct _XENVIF_POLLER_CHANNEL {
+ PXENVIF_POLLER_INSTANCE Instance;
+ XENVIF_POLLER_CHANNEL_TYPE Type;
+ const CHAR *Node;
+ PXENBUS_EVTCHN_CHANNEL Channel;
+ ULONG Events;
+} XENVIF_POLLER_CHANNEL, *PXENVIF_POLLER_CHANNEL;
+
+struct _XENVIF_POLLER_INSTANCE {
+ PXENVIF_POLLER Poller;
+ ULONG Index;
+ PCHAR Path;
+ KSPIN_LOCK Lock;
+ KDPC Dpc;
+ ULONG Dpcs;
+ KTIMER Timer;
+ KDPC TimerDpc;
+ PXENVIF_POLLER_CHANNEL Channel[XENVIF_POLLER_CHANNEL_TYPE_COUNT];
+ BOOLEAN Enabled;
+ LONG Pending;
+};
+
struct _XENVIF_POLLER {
PXENVIF_FRONTEND Frontend;
+ PXENVIF_POLLER_INSTANCE *Instance;
+ BOOLEAN Split;
+ XENBUS_STORE_INTERFACE StoreInterface;
+ XENBUS_EVTCHN_INTERFACE EvtchnInterface;
XENBUS_DEBUG_INTERFACE DebugInterface;
PXENBUS_DEBUG_CALLBACK DebugCallback;
};
-#define XENVIF_POLLER_TAG 'LLOP'
+#define XENVIF_POLLER_TAG 'LLOP'
+
+static FORCEINLINE PVOID
+__PollerAllocate(
+ IN ULONG Length
+ )
+{
+ return __AllocatePoolWithTag(NonPagedPool, Length, XENVIF_POLLER_TAG);
+}
+
+static FORCEINLINE VOID
+__PollerFree(
+ IN PVOID Buffer
+ )
+{
+ __FreePoolWithTag(Buffer, XENVIF_POLLER_TAG);
+}
+
+static NTSTATUS
+PollerChannelInitialize(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN ULONG Type,
+ OUT PXENVIF_POLLER_CHANNEL *Channel
+ )
+{
+ NTSTATUS status;
+
+ *Channel = __PollerAllocate(sizeof (XENVIF_POLLER_CHANNEL));
+
+ status = STATUS_NO_MEMORY;
+ if (*Channel == NULL)
+ goto fail1;
+
+ (*Channel)->Instance = Instance;
+ (*Channel)->Type = Type;
+
+ switch (Type) {
+ case XENVIF_POLLER_CHANNEL_RECEIVER:
+ (*Channel)->Node = "event-channel-rx";
+ break;
+
+ case XENVIF_POLLER_CHANNEL_TRANSMITTER:
+ (*Channel)->Node = "event-channel-tx";
+ break;
+
+ case XENVIF_POLLER_CHANNEL_COMBINED:
+ (*Channel)->Node = "event-channel";
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static VOID
+PollerChannelSetPending(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+
+ Instance = Channel->Instance;
+
+ switch (Channel->Type)
+ {
+ case XENVIF_POLLER_CHANNEL_RECEIVER:
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ break;
+
+ case XENVIF_POLLER_CHANNEL_TRANSMITTER:
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_TRANSMIT);
+ break;
+
+ case XENVIF_POLLER_CHANNEL_COMBINED:
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_TRANSMIT);
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+}
+
+static FORCEINLINE BOOLEAN
+__BitTest(
+ IN PLONG Mask,
+ IN LONG Bit
+ )
+{
+ return (*Mask & (1L << Bit)) ? TRUE : FALSE;
+}
+
+static BOOLEAN
+PollerChannelTestPending(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+
+ Instance = Channel->Instance;
+
+ switch (Channel->Type)
+ {
+ case XENVIF_POLLER_CHANNEL_RECEIVER:
+ if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_RECEIVE))
+ return TRUE;
+
+ break;
+
+ case XENVIF_POLLER_CHANNEL_TRANSMITTER:
+ if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_TRANSMIT))
+ return TRUE;
+
+ break;
+
+ case XENVIF_POLLER_CHANNEL_COMBINED:
+ if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_RECEIVE) ||
+ __BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_TRANSMIT))
+ return TRUE;
+
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+
+ return FALSE;
+}
+
+KSERVICE_ROUTINE PollerChannelEvtchnCallback;
+
+BOOLEAN
+PollerChannelEvtchnCallback(
+ IN PKINTERRUPT InterruptObject,
+ IN PVOID Argument
+ )
+{
+ PXENVIF_POLLER_CHANNEL Channel = Argument;
+ PXENVIF_POLLER_INSTANCE Instance;
+
+ UNREFERENCED_PARAMETER(InterruptObject);
+
+ ASSERT(Channel != NULL);
+ Instance = Channel->Instance;
+
+ Channel->Events++;
+
+ PollerChannelSetPending(Channel);
+
+ if (KeInsertQueueDpc(&Instance->Dpc, NULL, NULL))
+ Instance->Dpcs++;
+
+ return TRUE;
+}
+
+static FORCEINLINE BOOLEAN
+__PollerIsSplit(
+ IN PXENVIF_POLLER Poller
+ )
+{
+ return Poller->Split;
+}
+
+static NTSTATUS
+PollerChannelConnect(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+ PXENVIF_FRONTEND Frontend;
+ PROCESSOR_NUMBER ProcNumber;
+ NTSTATUS status;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+ Frontend = Poller->Frontend;
+
+ switch (Channel->Type)
+ {
+ case XENVIF_POLLER_CHANNEL_RECEIVER:
+ case XENVIF_POLLER_CHANNEL_TRANSMITTER:
+ if (!__PollerIsSplit(Poller))
+ goto done;
+
+ break;
+
+ case XENVIF_POLLER_CHANNEL_COMBINED:
+ if (__PollerIsSplit(Poller))
+ goto done;
+
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+
+ Channel->Channel = XENBUS_EVTCHN(Open,
+ &Poller->EvtchnInterface,
+ XENBUS_EVTCHN_TYPE_UNBOUND,
+ PollerChannelEvtchnCallback,
+ Channel,
+ FrontendGetBackendDomain(Frontend),
+ TRUE);
+
+ status = STATUS_UNSUCCESSFUL;
+ if (Channel->Channel == NULL)
+ goto fail1;
+
+ status = KeGetProcessorNumberFromIndex(Instance->Index, &ProcNumber);
+ ASSERT(NT_SUCCESS(status));
+
+ (VOID) XENBUS_EVTCHN(Bind,
+ &Poller->EvtchnInterface,
+ Channel->Channel,
+ ProcNumber.Group,
+ ProcNumber.Number);
+
+ XENBUS_EVTCHN(Unmask,
+ &Poller->EvtchnInterface,
+ Channel->Channel,
+ FALSE);
+
+done:
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static NTSTATUS
+PollerChannelStoreWrite(
+ IN PXENVIF_POLLER_CHANNEL Channel,
+ IN PXENBUS_STORE_TRANSACTION Transaction
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+ ULONG Port;
+ NTSTATUS status;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ if (Channel->Channel == NULL)
+ goto done;
+
+ Port = XENBUS_EVTCHN(GetPort,
+ &Poller->EvtchnInterface,
+ Channel->Channel);
+
+ status = XENBUS_STORE(Printf,
+ &Poller->StoreInterface,
+ Transaction,
+ Instance->Path,
+ (PCHAR)Channel->Node,
+ "%u",
+ Port);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+
+done:
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static VOID
+PollerChannelUnmask(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ if (Channel->Channel == NULL)
+ return;
+
+ if (!PollerChannelTestPending(Channel))
+ XENBUS_EVTCHN(Unmask,
+ &Poller->EvtchnInterface,
+ Channel->Channel,
+ FALSE);
+}
+
+static VOID
+PollerChannelSend(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ XENBUS_EVTCHN(Send,
+ &Poller->EvtchnInterface,
+ Channel->Channel);
+}
+
+static VOID
+PollerChannelDebugCallback(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ if (Channel->Channel == NULL)
+ return;
+
+ XENBUS_DEBUG(Printf,
+ &Poller->DebugInterface,
+ "[%s]: Events = %lu\n",
+ Channel->Node,
+ Channel->Events);
+}
+
+static VOID
+PollerChannelDisconnect(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ if (Channel->Channel == NULL)
+ return;
+
+ Channel->Events = 0;
+
+ XENBUS_EVTCHN(Close,
+ &Poller->EvtchnInterface,
+ Channel->Channel);
+ Channel->Channel = NULL;
+}
+
+static VOID
+PollerChannelTeardown(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ Channel->Node = NULL;
+
+ Channel->Type = 0;
+ Channel->Instance = NULL;
+
+ ASSERT(IsZeroMemory(Channel, sizeof (XENVIF_POLLER_CHANNEL)));
+ __PollerFree(Channel);
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static VOID
+PollerInstanceUnmask(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ )
+{
+ PXENVIF_POLLER Poller;
+ XENVIF_POLLER_CHANNEL_TYPE Type;
+ PXENVIF_POLLER_CHANNEL Channel;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Poller = Instance->Poller;
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+
+ if (!Instance->Enabled)
+ goto done;
+
+ if (!__PollerIsSplit(Poller)) {
+ Type = XENVIF_POLLER_CHANNEL_COMBINED;
+ } else {
+ switch (Event) {
+ case XENVIF_POLLER_EVENT_RECEIVE:
+ Type = XENVIF_POLLER_CHANNEL_RECEIVER;
+ break;
+
+ case XENVIF_POLLER_EVENT_TRANSMIT:
+ Type = XENVIF_POLLER_CHANNEL_TRANSMITTER;
+ break;
+
+ default:
+ Type = XENVIF_POLLER_CHANNEL_INVALID;
+ break;
+ }
+ }
+
+ ASSERT(Type != XENVIF_POLLER_CHANNEL_INVALID);
+
+ Channel = Instance->Channel[Type];
+
+ PollerChannelUnmask(Channel);
+
+done:
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+}
+
+#define TIME_US(_us) ((_us) * 10)
+#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
+#define TIME_S(_s) (TIME_MS((_s) * 1000))
+#define TIME_RELATIVE(_t) (-(_t))
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static VOID
+PollerInstanceDefer(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ LARGE_INTEGER Delay;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+
+ if (!Instance->Enabled)
+ goto done;
+
+ Delay.QuadPart = TIME_RELATIVE(TIME_US(100));
+ KeSetTimer(&Instance->Timer, Delay, &Instance->TimerDpc);
+
+done:
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+}
+
+static FORCEINLINE BOOLEAN
+PollerInstanceDpcTimeout(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ KDPC_WATCHDOG_INFORMATION Watchdog;
+ NTSTATUS status;
+
+ UNREFERENCED_PARAMETER(Instance);
+
+ RtlZeroMemory(&Watchdog, sizeof (Watchdog));
+
+ status = KeQueryDpcWatchdogInformation(&Watchdog);
+ ASSERT(NT_SUCCESS(status));
+
+ if (Watchdog.DpcTimeLimit == 0 ||
+ Watchdog.DpcWatchdogLimit == 0)
+ return FALSE;
+
+ if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) &&
+ Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2))
+ return FALSE;
+
+ return TRUE;
+}
+
+__drv_functionClass(KDEFERRED_ROUTINE)
+__drv_maxIRQL(DISPATCH_LEVEL)
+__drv_minIRQL(DISPATCH_LEVEL)
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__drv_sameIRQL
+static VOID
+PollerInstanceDpc(
+ IN PKDPC Dpc,
+ IN PVOID Context,
+ IN PVOID Argument1,
+ IN PVOID Argument2
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance = Context;
+ PXENVIF_POLLER Poller;
+ PXENVIF_FRONTEND Frontend;
+ BOOLEAN NeedReceiverPoll;
+ BOOLEAN NeedTransmitterPoll;
+
+ UNREFERENCED_PARAMETER(Dpc);
+ UNREFERENCED_PARAMETER(Argument1);
+ UNREFERENCED_PARAMETER(Argument2);
+
+ ASSERT(Instance != NULL);
+
+ Poller = Instance->Poller;
+ Frontend = Poller->Frontend;
+
+ NeedReceiverPoll = FALSE;
+ NeedTransmitterPoll = FALSE;
+
+ for (;;) {
+ NeedReceiverPoll |=
+ (InterlockedBitTestAndReset(&Instance->Pending,
+ XENVIF_POLLER_EVENT_RECEIVE) != 0) ?
+ TRUE :
+ FALSE;
+
+ NeedTransmitterPoll |=
+ (InterlockedBitTestAndReset(&Instance->Pending,
+ XENVIF_POLLER_EVENT_TRANSMIT) != 0) ?
+ TRUE :
+ FALSE;
+
+ if (!NeedReceiverPoll && !NeedTransmitterPoll)
+ break;
+
+ if (NeedReceiverPoll)
+ {
+ BOOLEAN Retry = ReceiverPoll(FrontendGetReceiver(Frontend),
+ Instance->Index);
+
+ if (!Retry) {
+ NeedReceiverPoll = FALSE;
+ PollerInstanceUnmask(Instance, XENVIF_POLLER_EVENT_RECEIVE);
+ }
+ }
+ if (NeedTransmitterPoll)
+ {
+ BOOLEAN Retry = TransmitterPoll(FrontendGetTransmitter(Frontend),
+ Instance->Index);
+
+ if (!Retry) {
+ NeedTransmitterPoll = FALSE;
+ PollerInstanceUnmask(Instance, XENVIF_POLLER_EVENT_TRANSMIT);
+ }
+ }
+
+ if (PollerInstanceDpcTimeout(Instance)) {
+ PollerInstanceDefer(Instance);
+ break;
+ }
+ }
+}
+
+static NTSTATUS
+PollerInstanceInitialize(
+ IN PXENVIF_POLLER Poller,
+ IN LONG Index,
+ OUT PXENVIF_POLLER_INSTANCE *Instance
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ LONG Type;
+ NTSTATUS status;
+
+ Frontend = Poller->Frontend;
+
+ *Instance = __PollerAllocate(sizeof (XENVIF_POLLER_INSTANCE));
+
+ status = STATUS_NO_MEMORY;
+ if (*Instance == NULL)
+ goto fail1;
+
+ (*Instance)->Poller = Poller;
+ (*Instance)->Index = Index;
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel;
+
+ status = PollerChannelInitialize(*Instance, Type, &Channel);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+ (*Instance)->Channel[Type] = Channel;
+ }
+
+ (*Instance)->Path = FrontendFormatPath(Frontend, Index);
+ if ((*Instance)->Path == NULL)
+ goto fail3;
+
+ KeInitializeSpinLock(&(*Instance)->Lock);
+
+ KeInitializeDpc(&(*Instance)->Dpc, PollerInstanceDpc, *Instance);
+ KeInitializeTimer(&(*Instance)->Timer);
+ KeInitializeDpc(&(*Instance)->TimerDpc, PollerInstanceDpc, *Instance);
+
+ return STATUS_SUCCESS;
+
+fail3:
+ Error("fail3\n");
+
+ Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
+
+fail2:
+ Error("fail2\n");
+
+ while (--Type >= 0)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = (*Instance)->Channel[Type];
+
+ (*Instance)->Channel[Type] = NULL;
+ PollerChannelTeardown(Channel);
+ }
+
+ (*Instance)->Index = 0;
+ (*Instance)->Poller = NULL;
+
+ ASSERT(IsZeroMemory(*Instance, sizeof (XENVIF_POLLER_INSTANCE)));
+ __PollerFree(*Instance);
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static NTSTATUS
+PollerInstanceConnect(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ PROCESSOR_NUMBER ProcNumber;
+ LONG Type;
+ NTSTATUS status;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ status = KeGetProcessorNumberFromIndex(Instance->Index, &ProcNumber);
+ ASSERT(NT_SUCCESS(status));
+
+ KeSetTargetProcessorDpcEx(&Instance->Dpc, &ProcNumber);
+ KeSetTargetProcessorDpcEx(&Instance->TimerDpc, &ProcNumber);
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ status = PollerChannelConnect(Channel);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+ }
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ while (--Type >= 0)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ PollerChannelDisconnect(Channel);
+ }
+
+ return status;
+}
+
+static NTSTATUS
+PollerInstanceStoreWrite(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN PXENBUS_STORE_TRANSACTION Transaction
+ )
+{
+ ULONG Type;
+ NTSTATUS status;
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ status = PollerChannelStoreWrite(Channel, Transaction);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+ }
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
-static FORCEINLINE PVOID
-__PollerAllocate(
- IN ULONG Length
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static NTSTATUS
+PollerInstanceEnable(
+ IN PXENVIF_POLLER_INSTANCE Instance
)
{
- return __AllocatePoolWithTag(NonPagedPool, Length, XENVIF_POLLER_TAG);
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_TRANSMIT);
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+ Instance->Enabled = TRUE;
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+
+ (VOID) KeInsertQueueDpc(&Instance->Dpc, NULL, NULL);
+
+ return STATUS_SUCCESS;
}
-static FORCEINLINE VOID
-__PollerFree(
- IN PVOID Buffer
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static NTSTATUS
+PollerInstanceSend(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN XENVIF_POLLER_EVENT_TYPE Event
)
{
- __FreePoolWithTag(Buffer, XENVIF_POLLER_TAG);
+ PXENVIF_POLLER Poller;
+ XENVIF_POLLER_CHANNEL_TYPE Type;
+ PXENVIF_POLLER_CHANNEL Channel;
+ NTSTATUS status;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Poller = Instance->Poller;
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+
+ Type = XENVIF_POLLER_CHANNEL_INVALID;
+
+ if (Instance->Enabled) {
+ if (!__PollerIsSplit(Poller)) {
+ Type = XENVIF_POLLER_CHANNEL_COMBINED;
+ } else {
+ switch (Event) {
+ case XENVIF_POLLER_EVENT_RECEIVE:
+ Type = XENVIF_POLLER_CHANNEL_RECEIVER;
+ break;
+
+ case XENVIF_POLLER_EVENT_TRANSMIT:
+ Type = XENVIF_POLLER_CHANNEL_TRANSMITTER;
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+ }
+ }
+
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+
+ status = STATUS_UNSUCCESSFUL;
+ if (Type == XENVIF_POLLER_CHANNEL_INVALID)
+ goto fail1;
+
+ Channel = Instance->Channel[Type];
+
+ PollerChannelSend(Channel);
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static NTSTATUS
+PollerInstanceTrigger(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ )
+{
+ NTSTATUS status;
+
+ status = STATUS_INVALID_PARAMETER;
+ if (Event >= XENVIF_POLLER_EVENT_TYPE_COUNT)
+ goto fail1;
+
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending, Event);
+
+ if (KeInsertQueueDpc(&Instance->Dpc, NULL, NULL))
+ Instance->Dpcs++;
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static VOID
+PollerInstanceDebugCallback(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ PXENVIF_POLLER Poller;
+ ULONG Type;
+
+ Poller = Instance->Poller;
+
+ XENBUS_DEBUG(Printf,
+ &Poller->DebugInterface,
+ "[%d]: Dpcs = %lu\n",
+ Instance->Index,
+ Instance->Dpcs);
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ PollerChannelDebugCallback(Channel);
+ }
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static VOID
+PollerInstanceDisable(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+ Instance->Enabled = FALSE;
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+
+ //
+ // No new timers can be scheduled once Enabled goes to FALSE.
+ // Cancel any existing ones.
+ //
+ (VOID) KeCancelTimer(&Instance->Timer);
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static VOID
+PollerInstanceDisconnect(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ LONG Type;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Instance->Dpcs = 0;
+ Instance->Pending = 0;
+
+ Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
+
+ while (--Type >= 0)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ PollerChannelDisconnect(Channel);
+ }
+}
+
+static VOID
+PollerInstanceTeardown(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ PXENVIF_POLLER Poller;
+ PXENVIF_FRONTEND Frontend;
+ LONG Type;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
+ KeFlushQueuedDpcs();
+
+ Poller = Instance->Poller;
+ Frontend = Poller->Frontend;
+
+ RtlZeroMemory(&Instance->TimerDpc, sizeof (KDPC));
+ RtlZeroMemory(&Instance->Timer, sizeof (KTIMER));
+ RtlZeroMemory(&Instance->Dpc, sizeof (KDPC));
+
+ RtlZeroMemory(&Instance->Lock, sizeof (KSPIN_LOCK));
+
+ FrontendFreePath(Frontend, Instance->Path);
+ Instance->Path = NULL;
+
+ Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
+
+ while (--Type >= 0)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ Instance->Channel[Type] = NULL;
+ PollerChannelTeardown(Channel);
+ }
+
+ Instance->Index = 0;
+ Instance->Poller = NULL;
+
+ ASSERT(IsZeroMemory(Instance, sizeof (XENVIF_POLLER_INSTANCE)));
+ __PollerFree(Instance);
}
static VOID
PollerDebugCallback(
- IN PVOID Argument,
- IN BOOLEAN Crashing
+ IN PVOID Argument,
+ IN BOOLEAN Crashing
)
{
- UNREFERENCED_PARAMETER(Argument);
+ PXENVIF_POLLER Poller = Argument;
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ ULONG Index;
+
UNREFERENCED_PARAMETER(Crashing);
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ for (Index = 0; Index < NumQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDebugCallback(Instance);
+ }
+}
+
+static VOID
+PollerSetSplit(
+ IN PXENVIF_POLLER Poller
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ PCHAR Buffer;
+ NTSTATUS status;
+
+ Frontend = Poller->Frontend;
+
+ status = XENBUS_STORE(Read,
+ &Poller->StoreInterface,
+ NULL,
+ FrontendGetBackendPath(Frontend),
+ "feature-split-event-channels",
+ &Buffer);
+ if (NT_SUCCESS(status)) {
+ Poller->Split = (BOOLEAN)strtol(Buffer, NULL, 2);
+
+ XENBUS_STORE(Free,
+ &Poller->StoreInterface,
+ Buffer);
+ } else {
+ Poller->Split = FALSE;
+ }
+
+ Info("%s: %s\n", FrontendGetPath(Frontend),
+ (Poller->Split) ? "TRUE" : "FALSE");
}
NTSTATUS
OUT PXENVIF_POLLER *Poller
)
{
+ LONG MaxQueues;
+ LONG Index;
NTSTATUS status;
*Poller = __PollerAllocate(sizeof (XENVIF_POLLER));
if (*Poller == NULL)
goto fail1;
+ FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
+ &(*Poller)->EvtchnInterface);
+
+ FdoGetStoreInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
+ &(*Poller)->StoreInterface);
+
FdoGetDebugInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
&(*Poller)->DebugInterface);
(*Poller)->Frontend = Frontend;
+ MaxQueues = FrontendGetMaxQueues(Frontend);
+ (*Poller)->Instance = __PollerAllocate(sizeof (PXENVIF_POLLER_INSTANCE) *
+ MaxQueues);
+
+ status = STATUS_NO_MEMORY;
+ if ((*Poller)->Instance == NULL)
+ goto fail2;
+
+ for (Index = 0; Index < MaxQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance;
+
+ status = PollerInstanceInitialize(*Poller, Index, &Instance);
+ if (!NT_SUCCESS(status))
+ goto fail3;
+
+ (*Poller)->Instance[Index] = Instance;
+ }
+
return STATUS_SUCCESS;
+fail3:
+ Error("fail3\n");
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = (*Poller)->Instance[Index];
+
+ (*Poller)->Instance[Index] = NULL;
+ PollerInstanceTeardown(Instance);
+ }
+
+ ASSERT(IsZeroMemory((*Poller)->Instance,
+ sizeof (PXENVIF_POLLER_INSTANCE) * MaxQueues));
+ __PollerFree((*Poller)->Instance);
+ (*Poller)->Instance = NULL;
+
+fail2:
+ Error("fail2\n");
+
+ (*Poller)->Frontend = NULL;
+
+ RtlZeroMemory(&(*Poller)->DebugInterface,
+ sizeof (XENBUS_DEBUG_INTERFACE));
+
+ RtlZeroMemory(&(*Poller)->StoreInterface,
+ sizeof (XENBUS_STORE_INTERFACE));
+
+ RtlZeroMemory(&(*Poller)->EvtchnInterface,
+ sizeof (XENBUS_EVTCHN_INTERFACE));
+
fail1:
Error("fail1 (%08x)\n", status);
IN PXENVIF_POLLER Poller
)
{
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
NTSTATUS status;
Trace("====>\n");
- status = XENBUS_DEBUG(Acquire, &Poller->DebugInterface);
+ Frontend = Poller->Frontend;
+
+ status = XENBUS_EVTCHN(Acquire, &Poller->EvtchnInterface);
if (!NT_SUCCESS(status))
goto fail1;
+ status = XENBUS_STORE(Acquire, &Poller->StoreInterface);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+ status = XENBUS_DEBUG(Acquire, &Poller->DebugInterface);
+ if (!NT_SUCCESS(status))
+ goto fail3;
+
+ PollerSetSplit(Poller);
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ for (Index = 0; Index < NumQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ status = PollerInstanceConnect(Instance);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+ }
+
status = XENBUS_DEBUG(Register,
&Poller->DebugInterface,
__MODULE__ "|POLLER",
Poller,
&Poller->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail2;
+ goto fail5;
Trace("<====\n");
return STATUS_SUCCESS;
+fail5:
+ Error("fail5\n");
+
+ Index = NumQueues;
+
+fail4:
+ Error("fail4\n");
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDisconnect(Instance);
+ }
+
+ Poller->Split = FALSE;
+
+ XENBUS_DEBUG(Release, &Poller->DebugInterface);
+
+fail3:
+ Error("fail3\n");
+
+ XENBUS_STORE(Release, &Poller->StoreInterface);
+
fail2:
Error("fail2\n");
- XENBUS_DEBUG(Release, &Poller->DebugInterface);
+ XENBUS_EVTCHN(Release, &Poller->EvtchnInterface);
fail1:
Error("fail1 (%08x)\n", status);
IN PXENBUS_STORE_TRANSACTION Transaction
)
{
- UNREFERENCED_PARAMETER(Poller);
- UNREFERENCED_PARAMETER(Transaction);
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
+ NTSTATUS status;
+
+ Trace("====>\n");
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ for (Index = 0; Index < NumQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
- Trace("<===>\n");
+ status = PollerInstanceStoreWrite(Instance, Transaction);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+ }
+
+ Trace("<====\n");
return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
}
NTSTATUS
PollerEnable(
- IN PXENVIF_POLLER Poller
+ IN PXENVIF_POLLER Poller
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
+ NTSTATUS status;
+
+ Trace("====>\n");
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ for (Index = 0; Index < NumQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ status = PollerInstanceEnable(Instance);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+ }
+
+ Trace("<====\n");
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDisable(Instance);
+ }
+
+ return status;
+}
+
+NTSTATUS
+PollerSend(
+ IN PXENVIF_POLLER Poller,
+ IN ULONG Index,
+ IN XENVIF_POLLER_EVENT_TYPE Event
)
{
- UNREFERENCED_PARAMETER(Poller);
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ PXENVIF_POLLER_INSTANCE Instance;
+ NTSTATUS status;
- Trace("<===>\n");
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ status = STATUS_INVALID_PARAMETER;
+ if (Index >= NumQueues)
+ goto fail1;
+
+ Instance = Poller->Instance[Index];
+
+ status = PollerInstanceSend(Instance, Event);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+ return STATUS_SUCCESS;
+
+fail2:
+ Error("fail2\n");
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+NTSTATUS
+PollerTrigger(
+ IN PXENVIF_POLLER Poller,
+ IN ULONG Index,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ PXENVIF_POLLER_INSTANCE Instance;
+ NTSTATUS status;
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ status = STATUS_INVALID_PARAMETER;
+ if (Index >= NumQueues)
+ goto fail1;
+
+ Instance = Poller->Instance[Index];
+
+ status = PollerInstanceTrigger(Instance, Event);
+ if (!NT_SUCCESS(status))
+ goto fail2;
return STATUS_SUCCESS;
+
+fail2:
+ Error("fail2\n");
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
}
VOID
PollerDisable(
- IN PXENVIF_POLLER Poller
+ IN PXENVIF_POLLER Poller
)
{
- UNREFERENCED_PARAMETER(Poller);
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
- Trace("<===>\n");
+ Trace("====>\n");
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+ Index = NumQueues;
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDisable(Instance);
+ }
+
+ Trace("<====\n");
}
VOID
IN PXENVIF_POLLER Poller
)
{
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
+
Trace("====>\n");
+ Frontend = Poller->Frontend;
+
XENBUS_DEBUG(Deregister,
&Poller->DebugInterface,
Poller->DebugCallback);
Poller->DebugCallback = NULL;
+ NumQueues = FrontendGetNumQueues(Frontend);
+ Index = NumQueues;
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDisconnect(Instance);
+ }
+
+ Poller->Split = FALSE;
+
XENBUS_DEBUG(Release, &Poller->DebugInterface);
+ XENBUS_STORE(Release, &Poller->StoreInterface);
+
+ XENBUS_EVTCHN(Release, &Poller->EvtchnInterface);
+
Trace("<====\n");
}
IN PXENVIF_POLLER Poller
)
{
+ PXENVIF_FRONTEND Frontend;
+ LONG MaxQueues;
+ LONG Index;
+
ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
+ Frontend = Poller->Frontend;
+
+ MaxQueues = FrontendGetMaxQueues(Frontend);
+ Index = MaxQueues;
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ Poller->Instance[Index] = NULL;
+ PollerInstanceTeardown(Instance);
+ }
+
+ ASSERT(IsZeroMemory(Poller->Instance,
+ sizeof (PXENVIF_POLLER_INSTANCE) * MaxQueues));
+ __PollerFree(Poller->Instance);
+ Poller->Instance = NULL;
+
+ Poller->Frontend = NULL;
+
RtlZeroMemory(&Poller->DebugInterface,
sizeof (XENBUS_DEBUG_INTERFACE));
+ RtlZeroMemory(&Poller->StoreInterface,
+ sizeof (XENBUS_STORE_INTERFACE));
+
+ RtlZeroMemory(&Poller->EvtchnInterface,
+ sizeof (XENBUS_EVTCHN_INTERFACE));
+
ASSERT(IsZeroMemory(Poller, sizeof (XENVIF_POLLER)));
__PollerFree(Poller);
}
typedef struct _XENVIF_POLLER XENVIF_POLLER, *PXENVIF_POLLER;
+typedef enum _XENVIF_POLLER_EVENT_TYPE {
+ XENVIF_POLLER_EVENT_RECEIVE,
+ XENVIF_POLLER_EVENT_TRANSMIT,
+ XENVIF_POLLER_EVENT_TYPE_COUNT
+} XENVIF_POLLER_EVENT_TYPE, *PXENVIF_POLLER_EVENT_TYPE;
+
extern NTSTATUS
PollerInitialize(
IN PXENVIF_FRONTEND Frontend,
- OUT PXENVIF_POLLER *Poller
+ OUT PXENVIF_POLLER *Poller
);
extern NTSTATUS
extern NTSTATUS
PollerStoreWrite(
- IN PXENVIF_POLLER Poller,
+ IN PXENVIF_POLLER Poller,
IN PXENBUS_STORE_TRANSACTION Transaction
);
IN PXENVIF_POLLER Poller
);
+extern NTSTATUS
+PollerSend(
+ IN PXENVIF_POLLER Poller,
+ IN ULONG Index,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ );
+
+extern NTSTATUS
+PollerTrigger(
+ IN PXENVIF_POLLER Poller,
+ IN ULONG Index,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ );
+
extern VOID
PollerDisable(
IN PXENVIF_POLLER Poller
#include <store_interface.h>
#include <cache_interface.h>
#include <gnttab_interface.h>
-#include <evtchn_interface.h>
#include "pdo.h"
#include "registry.h"
netif_rx_front_ring_t Front;
netif_rx_sring_t *Shared;
PXENBUS_GNTTAB_ENTRY Entry;
- PXENBUS_EVTCHN_CHANNEL Channel;
- KDPC Dpc;
- ULONG Dpcs;
- KTIMER Timer;
- KDPC TimerDpc;
- ULONG Events;
PXENVIF_RECEIVER_FRAGMENT Pending[XENVIF_RECEIVER_MAXIMUM_FRAGMENT_ID + 1];
ULONG RequestsPosted;
ULONG RequestsPushed;
PXENVIF_FRONTEND Frontend;
XENBUS_CACHE_INTERFACE CacheInterface;
XENBUS_GNTTAB_INTERFACE GnttabInterface;
- XENBUS_EVTCHN_INTERFACE EvtchnInterface;
PXENVIF_RECEIVER_RING *Ring;
LONG Loaned;
LONG Returned;
return Ring->Stopped;
}
-static FORCEINLINE VOID
-__ReceiverRingTrigger(
- IN PXENVIF_RECEIVER_RING Ring,
- IN BOOLEAN Locked
- )
-{
- PXENVIF_RECEIVER Receiver;
-
- Receiver = Ring->Receiver;
-
- if (!Locked)
- __ReceiverRingAcquireLock(Ring);
-
- if (Ring->Connected)
- (VOID) XENBUS_EVTCHN(Trigger,
- &Receiver->EvtchnInterface,
- Ring->Channel);
-
- if (!Locked)
- __ReceiverRingReleaseLock(Ring);
-}
-
-static FORCEINLINE VOID
-__ReceiverRingSend(
- IN PXENVIF_RECEIVER_RING Ring,
- IN BOOLEAN Locked
- )
-{
- PXENVIF_RECEIVER Receiver;
-
- Receiver = Ring->Receiver;
-
- if (!Locked)
- __ReceiverRingAcquireLock(Ring);
-
- if (Ring->Connected)
- (VOID) XENBUS_EVTCHN(Send,
- &Receiver->EvtchnInterface,
- Ring->Channel);
-
- if (!Locked)
- __ReceiverRingReleaseLock(Ring);
-}
-
static FORCEINLINE VOID
__ReceiverRingReturnPacket(
IN PXENVIF_RECEIVER_RING Ring,
IN BOOLEAN Locked
)
{
+ PXENVIF_RECEIVER Receiver;
+ PXENVIF_FRONTEND Frontend;
PMDL Mdl;
+ Receiver = Ring->Receiver;
+ Frontend = Receiver->Frontend;
+
Mdl = &Packet->Mdl;
while (Mdl != NULL) {
if (__ReceiverRingIsStopped(Ring)) {
__ReceiverRingStart(Ring);
- __ReceiverRingTrigger(Ring, TRUE);
+ PollerTrigger(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_RECEIVE);
}
if (!Locked)
#pragma warning (pop)
- if (Notify)
- __ReceiverRingSend(Ring, TRUE);
+ if (Notify) {
+ PXENVIF_RECEIVER Receiver;
+ PXENVIF_FRONTEND Frontend;
+
+ Receiver = Ring->Receiver;
+ Frontend = Receiver->Frontend;
+
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ }
Ring->RequestsPushed = Ring->RequestsPosted;
}
Ring->RequestsPosted,
Ring->RequestsPushed,
Ring->ResponsesProcessed);
-
- // Dump event channel
- XENBUS_DEBUG(Printf,
- &Receiver->DebugInterface,
- "[%s]: Events = %lu Dpcs = %lu\n",
- FrontendIsSplit(Frontend) ? "RX" : "COMBINED",
- Ring->Events,
- Ring->Dpcs);
}
static DECLSPEC_NOINLINE BOOLEAN
#undef XENVIF_RECEIVER_BATCH
}
-static FORCEINLINE VOID
-__ReceiverRingUnmask(
- IN PXENVIF_RECEIVER_RING Ring
- )
-{
- PXENVIF_RECEIVER Receiver;
-
- if (!Ring->Connected)
- return;
-
- Receiver = Ring->Receiver;
-
- XENBUS_EVTCHN(Unmask,
- &Receiver->EvtchnInterface,
- Ring->Channel,
- FALSE);
-}
-
-static FORCEINLINE BOOLEAN
-__ReceiverRingDpcTimeout(
- IN PXENVIF_RECEIVER_RING Ring
- )
-{
- KDPC_WATCHDOG_INFORMATION Watchdog;
- NTSTATUS status;
-
- UNREFERENCED_PARAMETER(Ring);
-
- RtlZeroMemory(&Watchdog, sizeof (Watchdog));
-
- status = KeQueryDpcWatchdogInformation(&Watchdog);
- ASSERT(NT_SUCCESS(status));
-
- if (Watchdog.DpcTimeLimit == 0 ||
- Watchdog.DpcWatchdogLimit == 0)
- return FALSE;
-
- if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) &&
- Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2))
- return FALSE;
-
- return TRUE;
-}
-
#define TIME_US(_us) ((_us) * 10)
#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
#define TIME_S(_s) (TIME_MS((_s) * 1000))
#define TIME_RELATIVE(_t) (-(_t))
-__drv_functionClass(KDEFERRED_ROUTINE)
-__drv_maxIRQL(DISPATCH_LEVEL)
-__drv_minIRQL(DISPATCH_LEVEL)
-__drv_requiresIRQL(DISPATCH_LEVEL)
-__drv_sameIRQL
-static VOID
-ReceiverRingDpc(
- IN PKDPC Dpc,
- IN PVOID Context,
- IN PVOID Argument1,
- IN PVOID Argument2
- )
-{
- PXENVIF_RECEIVER_RING Ring = Context;
-
- UNREFERENCED_PARAMETER(Dpc);
- UNREFERENCED_PARAMETER(Argument1);
- UNREFERENCED_PARAMETER(Argument2);
-
- ASSERT(Ring != NULL);
-
- for (;;) {
- BOOLEAN Retry;
-
- __ReceiverRingAcquireLock(Ring);
- Retry = ReceiverRingPoll(Ring);
- __ReceiverRingReleaseLock(Ring);
-
- if (!Retry) {
- __ReceiverRingUnmask(Ring);
- break;
- }
-
- if (__ReceiverRingDpcTimeout(Ring)) {
- LARGE_INTEGER Delay;
-
- Delay.QuadPart = TIME_RELATIVE(TIME_US(100));
-
- KeSetTimer(&Ring->Timer, Delay, &Ring->TimerDpc);
- break;
- }
- }
-}
-
-KSERVICE_ROUTINE ReceiverRingEvtchnCallback;
-
-BOOLEAN
-ReceiverRingEvtchnCallback(
- IN PKINTERRUPT InterruptObject,
- IN PVOID Argument
- )
-{
- PXENVIF_RECEIVER_RING Ring = Argument;
- PXENVIF_RECEIVER Receiver;
- PXENVIF_FRONTEND Frontend;
-
- UNREFERENCED_PARAMETER(InterruptObject);
-
- ASSERT(Ring != NULL);
-
- Ring->Events++;
-
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- Ring->Dpcs++;
-
- Receiver = Ring->Receiver;
- Frontend = Receiver->Frontend;
-
- if (!FrontendIsSplit(Frontend))
- TransmitterNotify(FrontendGetTransmitter(Frontend),
- Ring->Index);
-
- return TRUE;
-}
-
#define XENVIF_RECEIVER_WATCHDOG_PERIOD 30
static NTSTATUS
if (Ring->Shared->rsp_prod != rsp_prod &&
Ring->Front.rsp_cons == rsp_cons) {
PXENVIF_RECEIVER Receiver;
+ PXENVIF_FRONTEND Frontend;
Receiver = Ring->Receiver;
+ Frontend = Receiver->Frontend;
XENBUS_DEBUG(Trigger,
&Receiver->DebugInterface,
Ring->DebugCallback);
// Try to move things along
- __ReceiverRingTrigger(Ring, TRUE);
- __ReceiverRingSend(Ring, TRUE);
+ PollerTrigger(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_RECEIVE);
}
KeMemoryBarrier();
InitializeListHead(&(*Ring)->PacketList);
- KeInitializeDpc(&(*Ring)->Dpc, ReceiverRingDpc, *Ring);
- KeInitializeTimer(&(*Ring)->Timer);
- KeInitializeDpc(&(*Ring)->TimerDpc, ReceiverRingDpc, *Ring);
-
status = RtlStringCbPrintfA(Name,
sizeof (Name),
"%s_receiver_packet",
fail3:
Error("fail3\n");
- RtlZeroMemory(&(*Ring)->TimerDpc, sizeof (KDPC));
- RtlZeroMemory(&(*Ring)->Timer, sizeof (KTIMER));
- RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC));
-
RtlZeroMemory(&(*Ring)->PacketList, sizeof (LIST_ENTRY));
FrontendFreePath(Frontend, (*Ring)->Path);
PFN_NUMBER Pfn;
CHAR Name[MAXNAMELEN];
ULONG Index;
- PROCESSOR_NUMBER ProcNumber;
NTSTATUS status;
Receiver = Ring->Receiver;
ASSERT(!Ring->Connected);
- Ring->Channel = XENBUS_EVTCHN(Open,
- &Receiver->EvtchnInterface,
- XENBUS_EVTCHN_TYPE_UNBOUND,
- ReceiverRingEvtchnCallback,
- Ring,
- FrontendGetBackendDomain(Frontend),
- TRUE);
-
- status = STATUS_UNSUCCESSFUL;
- if (Ring->Channel == NULL)
- goto fail6;
-
- status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber);
- ASSERT(NT_SUCCESS(status));
-
- KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber);
- KeSetTargetProcessorDpcEx(&Ring->TimerDpc, &ProcNumber);
-
- (VOID) XENBUS_EVTCHN(Bind,
- &Receiver->EvtchnInterface,
- Ring->Channel,
- ProcNumber.Group,
- ProcNumber.Number);
-
- XENBUS_EVTCHN(Unmask,
- &Receiver->EvtchnInterface,
- Ring->Channel,
- FALSE);
-
Ring->Connected = TRUE;
status = XENBUS_DEBUG(Register,
Ring,
&Ring->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail7;
+ goto fail6;
return STATUS_SUCCESS;
-fail7:
- Error("fail7\n");
-
- Ring->Connected = FALSE;
-
- XENBUS_EVTCHN(Close,
- &Receiver->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- Ring->Events = 0;
-
fail6:
Error("fail6\n");
+ Ring->Connected = FALSE;
+
fail5:
Error("fail5\n");
{
PXENVIF_RECEIVER Receiver;
PXENVIF_FRONTEND Frontend;
- ULONG Port;
PCHAR Path;
NTSTATUS status;
if (!NT_SUCCESS(status))
goto fail1;
- Port = XENBUS_EVTCHN(GetPort,
- &Receiver->EvtchnInterface,
- Ring->Channel);
-
- status = XENBUS_STORE(Printf,
- &Receiver->StoreInterface,
- Transaction,
- Path,
- FrontendIsSplit(Frontend) ? "event-channel-rx" : "event-channel",
- "%u",
- Port);
- if (!NT_SUCCESS(status))
- goto fail2;
-
return STATUS_SUCCESS;
-fail2:
- Error("fail2\n");
-
fail1:
Error("fail1 (%08x)\n", status);
Ring->Enabled = TRUE;
- (VOID) KeInsertQueueDpc(&Ring->Dpc, NULL, NULL);
-
__ReceiverRingReleaseLock(Ring);
Info("%s[%u]: <====\n",
__ReceiverRingReleaseLock(Ring);
- //
- // No new timers can be scheduled once Enabled goes to FALSE.
- // Cancel any existing ones.
- //
- (VOID) KeCancelTimer(&Ring->Timer);
-
Info("%s[%u]: <====\n",
FrontendGetPath(Frontend),
Ring->Index);
ASSERT(Ring->Connected);
Ring->Connected = FALSE;
- XENBUS_EVTCHN(Close,
- &Receiver->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- Ring->Events = 0;
- Ring->Dpcs = 0;
-
ASSERT3U(Ring->ResponsesProcessed, ==, Ring->RequestsPushed);
ASSERT3U(Ring->RequestsPushed, ==, Ring->RequestsPosted);
Frontend = Receiver->Frontend;
RtlZeroMemory(&Ring->Hash, sizeof (XENVIF_RECEIVER_HASH));
- RtlZeroMemory(&Ring->TimerDpc, sizeof (KDPC));
- RtlZeroMemory(&Ring->Timer, sizeof (KTIMER));
- RtlZeroMemory(&Ring->Dpc, sizeof (KDPC));
Ring->BackfillSize = 0;
Ring->OffloadOptions.Value = 0;
FdoGetGnttabInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
&(*Receiver)->GnttabInterface);
- FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
- &(*Receiver)->EvtchnInterface);
-
(*Receiver)->Frontend = Frontend;
status = XENBUS_CACHE(Acquire, &(*Receiver)->CacheInterface);
(*Receiver)->Frontend = NULL;
- RtlZeroMemory(&(*Receiver)->EvtchnInterface,
- sizeof (XENBUS_EVTCHN_INTERFACE));
-
RtlZeroMemory(&(*Receiver)->GnttabInterface,
sizeof (XENBUS_GNTTAB_INTERFACE));
if (!NT_SUCCESS(status))
goto fail2;
- status = XENBUS_EVTCHN(Acquire, &Receiver->EvtchnInterface);
- if (!NT_SUCCESS(status))
- goto fail3;
-
status = XENBUS_GNTTAB(Acquire, &Receiver->GnttabInterface);
if (!NT_SUCCESS(status))
- goto fail4;
+ goto fail3;
Index = 0;
while (Index < (LONG)FrontendGetNumQueues(Frontend)) {
status = __ReceiverRingConnect(Ring);
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail4;
Index++;
}
Receiver,
&Receiver->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail6;
+ goto fail5;
Trace("<====\n");
return STATUS_SUCCESS;
-fail6:
- Error("fail6\n");
+fail5:
+ Error("fail5\n");
Index = FrontendGetNumQueues(Frontend);
-fail5:
- Error("fail5\n");
+fail4:
+ Error("fail4\n");
while (--Index >= 0) {
PXENVIF_RECEIVER_RING Ring = Receiver->Ring[Index];
XENBUS_GNTTAB(Release, &Receiver->GnttabInterface);
-fail4:
- Error("fail4\n");
-
- XENBUS_EVTCHN(Release, &Receiver->EvtchnInterface);
-
fail3:
Error("fail3\n");
return status;
}
+BOOLEAN
+ReceiverPoll(
+ IN PXENVIF_RECEIVER Receiver,
+ IN ULONG Index
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ PXENVIF_RECEIVER_RING Ring;
+ BOOLEAN Retry;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Frontend = Receiver->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+ if (Index >= NumQueues)
+ return FALSE;
+
+ Ring = Receiver->Ring[Index];
+
+ __ReceiverRingAcquireLock(Ring);
+ Retry = ReceiverRingPoll(Ring);
+ __ReceiverRingReleaseLock(Ring);
+
+ return Retry;
+}
+
VOID
ReceiverDisable(
IN PXENVIF_RECEIVER Receiver
XENBUS_GNTTAB(Release, &Receiver->GnttabInterface);
- XENBUS_EVTCHN(Release, &Receiver->EvtchnInterface);
-
XENBUS_STORE(Release, &Receiver->StoreInterface);
XENBUS_DEBUG(Release, &Receiver->DebugInterface);
Frontend = Receiver->Frontend;
- ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
- KeFlushQueuedDpcs();
-
ASSERT3U(Receiver->Returned, ==, Receiver->Loaned);
Receiver->Loaned = 0;
Receiver->Returned = 0;
Receiver->Frontend = NULL;
- RtlZeroMemory(&Receiver->EvtchnInterface,
- sizeof (XENBUS_EVTCHN_INTERFACE));
-
RtlZeroMemory(&Receiver->GnttabInterface,
sizeof (XENBUS_GNTTAB_INTERFACE));
Trace("%s: <====\n", FrontendGetPath(Frontend));
}
-VOID
-ReceiverTrigger(
- IN PXENVIF_RECEIVER Receiver,
- IN ULONG Index
- )
-{
- PXENVIF_RECEIVER_RING Ring;
-
- Ring = Receiver->Ring[Index];
-
- __ReceiverRingTrigger(Ring, FALSE);
-}
-
-VOID
-ReceiverSend(
- IN PXENVIF_RECEIVER Receiver,
- IN ULONG Index
- )
-{
- PXENVIF_RECEIVER_RING Ring;
-
- Ring = Receiver->Ring[Index];
-
- __ReceiverRingSend(Ring, FALSE);
-}
-
NTSTATUS
ReceiverSetHashAlgorithm(
IN PXENVIF_RECEIVER Receiver,
IN PXENVIF_RECEIVER Receiver
);
+extern BOOLEAN
+ReceiverPoll(
+ IN PXENVIF_RECEIVER Receiver,
+ IN ULONG Index
+ );
+
extern VOID
ReceiverDisable(
IN PXENVIF_RECEIVER Receiver
#include <cache_interface.h>
#include <gnttab_interface.h>
#include <range_set_interface.h>
-#include <evtchn_interface.h>
#include "pdo.h"
#include "frontend.h"
netif_tx_front_ring_t Front;
netif_tx_sring_t *Shared;
PXENBUS_GNTTAB_ENTRY Entry;
- PXENBUS_EVTCHN_CHANNEL Channel;
- KDPC Dpc;
- ULONG Dpcs;
- KTIMER Timer;
- KDPC TimerDpc;
- ULONG Events;
BOOLEAN Connected;
BOOLEAN Enabled;
BOOLEAN Stopped;
XENBUS_CACHE_INTERFACE CacheInterface;
XENBUS_GNTTAB_INTERFACE GnttabInterface;
XENBUS_RANGE_SET_INTERFACE RangeSetInterface;
- XENBUS_EVTCHN_INTERFACE EvtchnInterface;
PXENVIF_TRANSMITTER_RING *Ring;
BOOLEAN MulticastControl;
ULONG DisableIpVersion4Gso;
Ring->PacketsUnprepared,
Ring->PacketsSent,
Ring->PacketsCompleted);
-
- if (FrontendIsSplit(Frontend)) {
- // Dump event channel
- XENBUS_DEBUG(Printf,
- &Transmitter->DebugInterface,
- "Events = %lu Dpcs = %lu\n",
- Ring->Events,
- Ring->Dpcs);
- }
}
static BOOLEAN
}
static FORCEINLINE VOID
-__TransmitterRingTrigger(
- IN PXENVIF_TRANSMITTER_RING Ring
- )
-{
- PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_FRONTEND Frontend;
-
- Transmitter = Ring->Transmitter;
- Frontend = Transmitter->Frontend;
-
- if (!Ring->Connected)
- return;
-
- if (FrontendIsSplit(Frontend)) {
- ASSERT(Ring->Channel != NULL);
-
- (VOID) XENBUS_EVTCHN(Trigger,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
- } else {
- ReceiverTrigger(FrontendGetReceiver(Frontend),
- Ring->Index);
- }
-}
-
-static FORCEINLINE VOID
-__TransmitterRingSend(
+__TransmitterRingPushRequests(
IN PXENVIF_TRANSMITTER_RING Ring
)
{
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
+ BOOLEAN Notify;
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
- if (!Ring->Connected)
- return;
-
- if (FrontendIsSplit(Frontend)) {
- ASSERT(Ring->Channel != NULL);
-
- (VOID) XENBUS_EVTCHN(Send,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
- } else {
- ReceiverSend(FrontendGetReceiver(Frontend),
- Ring->Index);
- }
-}
-
-static FORCEINLINE VOID
-__TransmitterRingPushRequests(
- IN PXENVIF_TRANSMITTER_RING Ring
- )
-{
- BOOLEAN Notify;
-
if (Ring->RequestsPosted == Ring->RequestsPushed)
return;
#pragma warning (pop)
if (Notify)
- __TransmitterRingSend(Ring);
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_TRANSMIT);
Ring->RequestsPushed = Ring->RequestsPosted;
}
__TransmitterRingReleaseLock(Ring);
}
-static FORCEINLINE VOID
-__TransmitterRingUnmask(
- IN PXENVIF_TRANSMITTER_RING Ring
- )
-{
- PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_FRONTEND Frontend;
-
- Transmitter = Ring->Transmitter;
- Frontend = Transmitter->Frontend;
-
- if (!Ring->Connected || !FrontendIsSplit(Frontend))
- return;
-
- XENBUS_EVTCHN(Unmask,
- &Transmitter->EvtchnInterface,
- Ring->Channel,
- FALSE);
-}
-
-static FORCEINLINE BOOLEAN
-__TransmitterRingDpcTimeout(
- IN PXENVIF_TRANSMITTER_RING Ring
- )
-{
- KDPC_WATCHDOG_INFORMATION Watchdog;
- NTSTATUS status;
-
- UNREFERENCED_PARAMETER(Ring);
-
- RtlZeroMemory(&Watchdog, sizeof (Watchdog));
-
- status = KeQueryDpcWatchdogInformation(&Watchdog);
- ASSERT(NT_SUCCESS(status));
-
- if (Watchdog.DpcTimeLimit == 0 ||
- Watchdog.DpcWatchdogLimit == 0)
- return FALSE;
-
- if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) &&
- Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2))
- return FALSE;
-
- return TRUE;
-}
-
#define TIME_US(_us) ((_us) * 10)
#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
#define TIME_S(_s) (TIME_MS((_s) * 1000))
#define TIME_RELATIVE(_t) (-(_t))
-__drv_functionClass(KDEFERRED_ROUTINE)
-__drv_maxIRQL(DISPATCH_LEVEL)
-__drv_minIRQL(DISPATCH_LEVEL)
-__drv_requiresIRQL(DISPATCH_LEVEL)
-__drv_sameIRQL
-static VOID
-TransmitterRingDpc(
- IN PKDPC Dpc,
- IN PVOID Context,
- IN PVOID Argument1,
- IN PVOID Argument2
- )
-{
- PXENVIF_TRANSMITTER_RING Ring = Context;
-
- UNREFERENCED_PARAMETER(Dpc);
- UNREFERENCED_PARAMETER(Argument1);
- UNREFERENCED_PARAMETER(Argument2);
-
- ASSERT(Ring != NULL);
-
- for (;;) {
- BOOLEAN Retry;
-
- __TransmitterRingAcquireLock(Ring);
- Retry = TransmitterRingPoll(Ring);
- __TransmitterRingReleaseLock(Ring);
-
- if (!Retry) {
- __TransmitterRingUnmask(Ring);
- break;
- }
-
- if (__TransmitterRingDpcTimeout(Ring)) {
- LARGE_INTEGER Delay;
-
- Delay.QuadPart = TIME_RELATIVE(TIME_US(100));
-
- KeSetTimer(&Ring->Timer, Delay, &Ring->TimerDpc);
- break;
- }
- }
-}
-
-KSERVICE_ROUTINE TransmitterRingEvtchnCallback;
-
-BOOLEAN
-TransmitterRingEvtchnCallback(
- IN PKINTERRUPT InterruptObject,
- IN PVOID Argument
- )
-{
- PXENVIF_TRANSMITTER_RING Ring = Argument;
- PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_FRONTEND Frontend;
-
- UNREFERENCED_PARAMETER(InterruptObject);
-
- ASSERT(Ring != NULL);
-
- Transmitter = Ring->Transmitter;
- Frontend = Transmitter->Frontend;
-
- ASSERT(FrontendIsSplit(Frontend));
-
- Ring->Events++;
-
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- Ring->Dpcs++;
-
- return TRUE;
-}
-
#define XENVIF_TRANSMITTER_WATCHDOG_PERIOD 30
static NTSTATUS
if (Ring->PacketsQueued == PacketsQueued &&
Ring->PacketsCompleted != PacketsQueued) {
PXENVIF_TRANSMITTER Transmitter;
+ PXENVIF_FRONTEND Frontend;
Transmitter = Ring->Transmitter;
+ Frontend = Transmitter->Frontend;
XENBUS_DEBUG(Trigger,
&Transmitter->DebugInterface,
Ring->DebugCallback);
// Try to move things along
- __TransmitterRingTrigger(Ring);
- __TransmitterRingSend(Ring);
+ PollerTrigger(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_TRANSMIT);
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_TRANSMIT);
}
PacketsQueued = Ring->PacketsQueued;
InitializeListHead(&(*Ring)->RequestQueue);
InitializeListHead(&(*Ring)->PacketComplete);
- KeInitializeDpc(&(*Ring)->Dpc, TransmitterRingDpc, *Ring);
- KeInitializeTimer(&(*Ring)->Timer);
- KeInitializeDpc(&(*Ring)->TimerDpc, TransmitterRingDpc, *Ring);
-
status = RtlStringCbPrintfA(Name,
sizeof (Name),
"%s_transmitter_buffer",
fail3:
Error("fail3\n");
- RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC));
-
RtlZeroMemory(&(*Ring)->PacketComplete, sizeof (LIST_ENTRY));
RtlZeroMemory(&(*Ring)->RequestQueue, sizeof (LIST_ENTRY));
RtlZeroMemory(&(*Ring)->PacketQueue, sizeof (LIST_ENTRY));
PFN_NUMBER Pfn;
CHAR Name[MAXNAMELEN];
ULONG Index;
- PROCESSOR_NUMBER ProcNumber;
NTSTATUS status;
ASSERT(!Ring->Connected);
ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
- if (FrontendIsSplit(Frontend)) {
- Ring->Channel = XENBUS_EVTCHN(Open,
- &Transmitter->EvtchnInterface,
- XENBUS_EVTCHN_TYPE_UNBOUND,
- TransmitterRingEvtchnCallback,
- Ring,
- FrontendGetBackendDomain(Frontend),
- TRUE);
-
- status = STATUS_UNSUCCESSFUL;
- if (Ring->Channel == NULL)
- goto fail6;
-
- status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber);
- ASSERT(NT_SUCCESS(status));
-
- KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber);
- KeSetTargetProcessorDpcEx(&Ring->TimerDpc, &ProcNumber);
-
- (VOID) XENBUS_EVTCHN(Bind,
- &Transmitter->EvtchnInterface,
- Ring->Channel,
- ProcNumber.Group,
- ProcNumber.Number);
-
- XENBUS_EVTCHN(Unmask,
- &Transmitter->EvtchnInterface,
- Ring->Channel,
- FALSE);
- }
-
status = XENBUS_DEBUG(Register,
&Transmitter->DebugInterface,
Name,
Ring,
&Ring->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail7;
+ goto fail6;
Ring->Connected = TRUE;
return STATUS_SUCCESS;
-fail7:
- Error("fail7\n");
-
- XENBUS_EVTCHN(Close,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- Ring->Events = 0;
-
fail6:
Error("fail6\n");
{
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
- ULONG Port;
PCHAR Path;
NTSTATUS status;
if (!NT_SUCCESS(status))
goto fail1;
- if (!FrontendIsSplit(Frontend))
- goto done;
-
- Port = XENBUS_EVTCHN(GetPort,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
-
- status = XENBUS_STORE(Printf,
- &Transmitter->StoreInterface,
- Transaction,
- Path,
- "event-channel-tx",
- "%u",
- Port);
- if (!NT_SUCCESS(status))
- goto fail2;
-
-done:
return STATUS_SUCCESS;
-fail2:
- Error("fail2\n");
-
fail1:
Error("fail1 (%08x)\n", status);
ASSERT(!Ring->Enabled);
Ring->Enabled = TRUE;
- KeInsertQueueDpc(&Ring->Dpc, NULL, NULL);
-
__TransmitterRingReleaseLock(Ring);
Info("%s[%u]: <====\n",
ASSERT(Attempt < 100);
// Try to move things along
- __TransmitterRingSend(Ring);
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_TRANSMIT);
(VOID) TransmitterRingPoll(Ring);
if (State != XenbusStateConnected)
__TransmitterRingReleaseLock(Ring);
- //
- // No new timers can be scheduled once Enabled goes to FALSE.
- // Cancel any existing ones.
- //
- (VOID) KeCancelTimer(&Ring->Timer);
-
Info("%s[%u]: <====\n",
FrontendGetPath(Frontend),
Ring->Index);
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
- if (Ring->Channel != NULL) {
- XENBUS_EVTCHN(Close,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- Ring->Events = 0;
- }
-
- Ring->Dpcs = 0;
-
ASSERT3U(Ring->ResponsesProcessed, ==, Ring->RequestsPushed);
ASSERT3U(Ring->RequestsPushed, ==, Ring->RequestsPosted);
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
- Ring->Dpcs = 0;
-
- RtlZeroMemory(&Ring->TimerDpc, sizeof (KDPC));
- RtlZeroMemory(&Ring->Timer, sizeof (KTIMER));
- RtlZeroMemory(&Ring->Dpc, sizeof (KDPC));
-
ASSERT3U(Ring->PacketsCompleted, ==, Ring->PacketsSent);
ASSERT3U(Ring->PacketsSent, ==, Ring->PacketsPrepared - Ring->PacketsUnprepared);
ASSERT3U(Ring->PacketsPrepared, ==, Ring->PacketsCopied + Ring->PacketsGranted + Ring->PacketsFaked);
FdoGetGnttabInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
&(*Transmitter)->GnttabInterface);
- FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
- &(*Transmitter)->EvtchnInterface);
-
(*Transmitter)->Frontend = Frontend;
KeInitializeSpinLock(&(*Transmitter)->Lock);
if (!NT_SUCCESS(status))
goto fail2;
- status = XENBUS_EVTCHN(Acquire, &Transmitter->EvtchnInterface);
- if (!NT_SUCCESS(status))
- goto fail3;
-
status = XENBUS_GNTTAB(Acquire, &Transmitter->GnttabInterface);
if (!NT_SUCCESS(status))
- goto fail4;
+ goto fail3;
if (Transmitter->DisableMulticastControl == 0) {
status = XENBUS_STORE(Read,
status = __TransmitterRingConnect(Ring);
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail4;
Index++;
}
Transmitter,
&Transmitter->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail6;
+ goto fail5;
Trace("<====\n");
return STATUS_SUCCESS;
-fail6:
- Error("fail6\n");
+fail5:
+ Error("fail5\n");
Index = FrontendGetNumQueues(Frontend);
-fail5:
- Error("fail5\n");
+fail4:
+ Error("fail4\n");
while (--Index >= 0) {
PXENVIF_TRANSMITTER_RING Ring;
XENBUS_GNTTAB(Release, &Transmitter->GnttabInterface);
-fail4:
- Error("fail4\n");
-
- XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface);
-
fail3:
Error("fail3\n");
return STATUS_SUCCESS;
}
+BOOLEAN
+TransmitterPoll(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN ULONG Index
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ PXENVIF_TRANSMITTER_RING Ring;
+ BOOLEAN Retry;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Frontend = Transmitter->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+ if (Index >= NumQueues)
+ return FALSE;
+
+ Ring = Transmitter->Ring[Index];
+
+ __TransmitterRingAcquireLock(Ring);
+ Retry = TransmitterRingPoll(Ring);
+ __TransmitterRingReleaseLock(Ring);
+
+ return Retry;
+}
+
VOID
TransmitterDisable(
IN PXENVIF_TRANSMITTER Transmitter
XENBUS_GNTTAB(Release, &Transmitter->GnttabInterface);
- XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface);
-
XENBUS_STORE(Release, &Transmitter->StoreInterface);
XENBUS_DEBUG(Release, &Transmitter->DebugInterface);
Frontend = Transmitter->Frontend;
ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
- KeFlushQueuedDpcs();
Index = FrontendGetMaxQueues(Frontend);
while (--Index >= 0) {
RtlZeroMemory(&Transmitter->DebugInterface,
sizeof (XENBUS_DEBUG_INTERFACE));
- RtlZeroMemory(&Transmitter->EvtchnInterface,
- sizeof (XENBUS_EVTCHN_INTERFACE));
-
Transmitter->DisableIpVersion4Gso = 0;
Transmitter->DisableIpVersion6Gso = 0;
Transmitter->AlwaysCopy = 0;
*Size = XENVIF_TRANSMITTER_RING_SIZE;
}
-VOID
-TransmitterNotify(
- IN PXENVIF_TRANSMITTER Transmitter,
- IN ULONG Index
- )
-{
- PXENVIF_FRONTEND Frontend;
- PXENVIF_TRANSMITTER_RING Ring;
-
- Frontend = Transmitter->Frontend;
-
- ASSERT(!FrontendIsSplit(Frontend));
-
- Ring = Transmitter->Ring[Index];
-
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- Ring->Dpcs++;
-}
-
VOID
TransmitterQueryOffloadOptions(
IN PXENVIF_TRANSMITTER Transmitter,
IN PXENVIF_TRANSMITTER Transmitter
);
+extern BOOLEAN
+TransmitterPoll(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN ULONG Index
+ );
+
extern VOID
TransmitterDisable(
IN PXENVIF_TRANSMITTER Transmitter
projects / pvdrivers / win / xenvif.git / commitdiff