USHORT BackendDomain;
ULONG MaxQueues;
ULONG NumQueues;
- BOOLEAN Split;
ULONG DisableToeplitz;
PXENVIF_MAC Mac;
return __FrontendGetNumQueues(Frontend);
}
-static VOID
-FrontendSetSplit(
- IN PXENVIF_FRONTEND Frontend
- )
-{
- PCHAR Buffer;
- NTSTATUS status;
-
- status = XENBUS_STORE(Read,
- &Frontend->StoreInterface,
- NULL,
- __FrontendGetBackendPath(Frontend),
- "feature-split-event-channels",
- &Buffer);
- if (NT_SUCCESS(status)) {
- Frontend->Split = (BOOLEAN)strtol(Buffer, NULL, 2);
-
- XENBUS_STORE(Free,
- &Frontend->StoreInterface,
- Buffer);
- } else {
- Frontend->Split = FALSE;
- }
-
- Info("%s: %s\n", __FrontendGetPath(Frontend),
- (Frontend->Split) ? "TRUE" : "FALSE");
-}
-
-static FORCEINLINE BOOLEAN
-__FrontendIsSplit(
- IN PXENVIF_FRONTEND Frontend
- )
-{
- return Frontend->Split;
-}
-
-BOOLEAN
-FrontendIsSplit(
- IN PXENVIF_FRONTEND Frontend
- )
-{
- return __FrontendIsSplit(Frontend);
-}
-
static FORCEINLINE NTSTATUS
__FrontendUpdateHash(
PXENVIF_FRONTEND Frontend,
goto fail3;
FrontendSetNumQueues(Frontend);
- FrontendSetSplit(Frontend);
status = PollerConnect(__FrontendGetPoller(Frontend));
if (!NT_SUCCESS(status))
MacDisconnect(__FrontendGetMac(Frontend));
- Frontend->Split = FALSE;
Frontend->NumQueues = 0;
fail3:
PollerDisconnect(__FrontendGetPoller(Frontend));
MacDisconnect(__FrontendGetMac(Frontend));
- Frontend->Split = FALSE;
Frontend->NumQueues = 0;
XENBUS_DEBUG(Deregister,
IN PXENVIF_FRONTEND Frontend
);
-extern BOOLEAN
-FrontendIsSplit(
- IN PXENVIF_FRONTEND Frontend
- );
-
-extern BOOLEAN
-FrontendIsSplit(
- IN PXENVIF_FRONTEND Frontend
- );
-
extern PCHAR
FrontendFormatPath(
IN PXENVIF_FRONTEND Frontend,
#include "pdo.h"
#include "frontend.h"
+#include "transmitter.h"
+#include "receiver.h"
#include "poller.h"
#include "vif.h"
#include "thread.h"
#define MAXNAMELEN 128
+typedef struct _XENVIF_POLLER_INSTANCE XENVIF_POLLER_INSTANCE, *PXENVIF_POLLER_INSTANCE;
+
+typedef enum _XENVIF_POLLER_CHANNEL_TYPE {
+ XENVIF_POLLER_CHANNEL_RECEIVER,
+ XENVIF_POLLER_CHANNEL_TRANSMITTER,
+ XENVIF_POLLER_CHANNEL_COMBINED,
+ XENVIF_POLLER_CHANNEL_TYPE_COUNT
+} XENVIF_POLLER_CHANNEL_TYPE, *PXENVIF_POLLER_CHANNEL_TYPE;
+
+#define XENVIF_POLLER_CHANNEL_INVALID XENVIF_POLLER_CHANNEL_TYPE_COUNT
+
+typedef struct _XENVIF_POLLER_CHANNEL {
+ PXENVIF_POLLER_INSTANCE Instance;
+ XENVIF_POLLER_CHANNEL_TYPE Type;
+ const CHAR *Node;
+ PXENBUS_EVTCHN_CHANNEL Channel;
+ ULONG Events;
+} XENVIF_POLLER_CHANNEL, *PXENVIF_POLLER_CHANNEL;
+
+struct _XENVIF_POLLER_INSTANCE {
+ PXENVIF_POLLER Poller;
+ ULONG Index;
+ PCHAR Path;
+ KSPIN_LOCK Lock;
+ KDPC Dpc;
+ ULONG Dpcs;
+ KTIMER Timer;
+ KDPC TimerDpc;
+ PXENVIF_POLLER_CHANNEL Channel[XENVIF_POLLER_CHANNEL_TYPE_COUNT];
+ BOOLEAN Enabled;
+ LONG Pending;
+};
+
struct _XENVIF_POLLER {
PXENVIF_FRONTEND Frontend;
+ PXENVIF_POLLER_INSTANCE *Instance;
+ BOOLEAN Split;
+ XENBUS_STORE_INTERFACE StoreInterface;
+ XENBUS_EVTCHN_INTERFACE EvtchnInterface;
XENBUS_DEBUG_INTERFACE DebugInterface;
PXENBUS_DEBUG_CALLBACK DebugCallback;
};
-#define XENVIF_POLLER_TAG 'LLOP'
+#define XENVIF_POLLER_TAG 'LLOP'
+
+static FORCEINLINE PVOID
+__PollerAllocate(
+ IN ULONG Length
+ )
+{
+ return __AllocatePoolWithTag(NonPagedPool, Length, XENVIF_POLLER_TAG);
+}
+
+static FORCEINLINE VOID
+__PollerFree(
+ IN PVOID Buffer
+ )
+{
+ __FreePoolWithTag(Buffer, XENVIF_POLLER_TAG);
+}
+
+static NTSTATUS
+PollerChannelInitialize(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN ULONG Type,
+ OUT PXENVIF_POLLER_CHANNEL *Channel
+ )
+{
+ NTSTATUS status;
+
+ *Channel = __PollerAllocate(sizeof (XENVIF_POLLER_CHANNEL));
+
+ status = STATUS_NO_MEMORY;
+ if (*Channel == NULL)
+ goto fail1;
+
+ (*Channel)->Instance = Instance;
+ (*Channel)->Type = Type;
+
+ switch (Type) {
+ case XENVIF_POLLER_CHANNEL_RECEIVER:
+ (*Channel)->Node = "event-channel-rx";
+ break;
+
+ case XENVIF_POLLER_CHANNEL_TRANSMITTER:
+ (*Channel)->Node = "event-channel-tx";
+ break;
+
+ case XENVIF_POLLER_CHANNEL_COMBINED:
+ (*Channel)->Node = "event-channel";
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static VOID
+PollerChannelSetPending(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+
+ Instance = Channel->Instance;
+
+ switch (Channel->Type)
+ {
+ case XENVIF_POLLER_CHANNEL_RECEIVER:
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ break;
+
+ case XENVIF_POLLER_CHANNEL_TRANSMITTER:
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_TRANSMIT);
+ break;
+
+ case XENVIF_POLLER_CHANNEL_COMBINED:
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_TRANSMIT);
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+}
+
+static FORCEINLINE BOOLEAN
+__BitTest(
+ IN PLONG Mask,
+ IN LONG Bit
+ )
+{
+ return (*Mask & (1L << Bit)) ? TRUE : FALSE;
+}
+
+static BOOLEAN
+PollerChannelTestPending(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+
+ Instance = Channel->Instance;
+
+ switch (Channel->Type)
+ {
+ case XENVIF_POLLER_CHANNEL_RECEIVER:
+ if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_RECEIVE))
+ return TRUE;
+
+ break;
+
+ case XENVIF_POLLER_CHANNEL_TRANSMITTER:
+ if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_TRANSMIT))
+ return TRUE;
+
+ break;
+
+ case XENVIF_POLLER_CHANNEL_COMBINED:
+ if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_RECEIVE) ||
+ __BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_TRANSMIT))
+ return TRUE;
+
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+
+ return FALSE;
+}
+
+KSERVICE_ROUTINE PollerChannelEvtchnCallback;
+
+BOOLEAN
+PollerChannelEvtchnCallback(
+ IN PKINTERRUPT InterruptObject,
+ IN PVOID Argument
+ )
+{
+ PXENVIF_POLLER_CHANNEL Channel = Argument;
+ PXENVIF_POLLER_INSTANCE Instance;
+
+ UNREFERENCED_PARAMETER(InterruptObject);
+
+ Instance = Channel->Instance;
+
+ ASSERT(Channel != NULL);
+ Channel->Events++;
+
+ PollerChannelSetPending(Channel);
+
+ if (KeInsertQueueDpc(&Instance->Dpc, NULL, NULL))
+ Instance->Dpcs++;
+
+ return TRUE;
+}
+
+static FORCEINLINE BOOLEAN
+__PollerIsSplit(
+ IN PXENVIF_POLLER Poller
+ )
+{
+ return Poller->Split;
+}
+
+static NTSTATUS
+PollerChannelConnect(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+ PXENVIF_FRONTEND Frontend;
+ PROCESSOR_NUMBER ProcNumber;
+ NTSTATUS status;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+ Frontend = Poller->Frontend;
+
+ switch (Channel->Type)
+ {
+ case XENVIF_POLLER_CHANNEL_RECEIVER:
+ case XENVIF_POLLER_CHANNEL_TRANSMITTER:
+ if (!__PollerIsSplit(Poller))
+ goto done;
+
+ break;
+
+ case XENVIF_POLLER_CHANNEL_COMBINED:
+ if (__PollerIsSplit(Poller))
+ goto done;
+
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+
+ Channel->Channel = XENBUS_EVTCHN(Open,
+ &Poller->EvtchnInterface,
+ XENBUS_EVTCHN_TYPE_UNBOUND,
+ PollerChannelEvtchnCallback,
+ Channel,
+ FrontendGetBackendDomain(Frontend),
+ TRUE);
+
+ status = STATUS_UNSUCCESSFUL;
+ if (Channel->Channel == NULL)
+ goto fail1;
+
+ status = KeGetProcessorNumberFromIndex(Instance->Index, &ProcNumber);
+ ASSERT(NT_SUCCESS(status));
+
+ (VOID) XENBUS_EVTCHN(Bind,
+ &Poller->EvtchnInterface,
+ Channel->Channel,
+ ProcNumber.Group,
+ ProcNumber.Number);
+
+ XENBUS_EVTCHN(Unmask,
+ &Poller->EvtchnInterface,
+ Channel->Channel,
+ FALSE);
+
+done:
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static NTSTATUS
+PollerChannelStoreWrite(
+ IN PXENVIF_POLLER_CHANNEL Channel,
+ IN PXENBUS_STORE_TRANSACTION Transaction
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+ ULONG Port;
+ NTSTATUS status;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ if (Channel->Channel == NULL)
+ goto done;
+
+ Port = XENBUS_EVTCHN(GetPort,
+ &Poller->EvtchnInterface,
+ Channel->Channel);
+
+ status = XENBUS_STORE(Printf,
+ &Poller->StoreInterface,
+ Transaction,
+ Instance->Path,
+ (PCHAR)Channel->Node,
+ "%u",
+ Port);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+
+done:
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static VOID
+PollerChannelUnmask(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ if (Channel->Channel == NULL)
+ return;
+
+ if (!PollerChannelTestPending(Channel))
+ XENBUS_EVTCHN(Unmask,
+ &Poller->EvtchnInterface,
+ Channel->Channel,
+ FALSE);
+}
+
+static VOID
+PollerChannelSend(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ XENBUS_EVTCHN(Send,
+ &Poller->EvtchnInterface,
+ Channel->Channel);
+}
+
+static VOID
+PollerChannelDebugCallback(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ if (Channel->Channel == NULL)
+ return;
+
+ XENBUS_DEBUG(Printf,
+ &Poller->DebugInterface,
+ "[%s]: Events = %lu\n",
+ Channel->Node,
+ Channel->Events);
+}
+
+static VOID
+PollerChannelDisconnect(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance;
+ PXENVIF_POLLER Poller;
+
+ Instance = Channel->Instance;
+ Poller = Instance->Poller;
+
+ if (Channel->Channel == NULL)
+ return;
+
+ Channel->Events = 0;
+
+ XENBUS_EVTCHN(Close,
+ &Poller->EvtchnInterface,
+ Channel->Channel);
+ Channel->Channel = NULL;
+}
+
+static VOID
+PollerChannelTeardown(
+ IN PXENVIF_POLLER_CHANNEL Channel
+ )
+{
+ Channel->Node = NULL;
+
+ Channel->Type = 0;
+ Channel->Instance = NULL;
+
+ ASSERT(IsZeroMemory(Channel, sizeof (XENVIF_POLLER_CHANNEL)));
+ __PollerFree(Channel);
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static VOID
+PollerInstanceUnmask(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ ULONG Type;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+
+ if (!Instance->Enabled)
+ goto done;
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ PollerChannelUnmask(Channel);
+ }
+
+done:
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+}
+
+#define TIME_US(_us) ((_us) * 10)
+#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
+#define TIME_S(_s) (TIME_MS((_s) * 1000))
+#define TIME_RELATIVE(_t) (-(_t))
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static VOID
+PollerInstanceDefer(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ LARGE_INTEGER Delay;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+
+ if (!Instance->Enabled)
+ goto done;
+
+ Delay.QuadPart = TIME_RELATIVE(TIME_US(100));
+ KeSetTimer(&Instance->Timer, Delay, &Instance->TimerDpc);
+
+done:
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+}
+
+static FORCEINLINE BOOLEAN
+PollerInstanceDpcTimeout(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ KDPC_WATCHDOG_INFORMATION Watchdog;
+ NTSTATUS status;
+
+ UNREFERENCED_PARAMETER(Instance);
+
+ RtlZeroMemory(&Watchdog, sizeof (Watchdog));
+
+ status = KeQueryDpcWatchdogInformation(&Watchdog);
+ ASSERT(NT_SUCCESS(status));
+
+ if (Watchdog.DpcTimeLimit == 0 ||
+ Watchdog.DpcWatchdogLimit == 0)
+ return FALSE;
+
+ if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) &&
+ Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2))
+ return FALSE;
+
+ return TRUE;
+}
+
+__drv_functionClass(KDEFERRED_ROUTINE)
+__drv_maxIRQL(DISPATCH_LEVEL)
+__drv_minIRQL(DISPATCH_LEVEL)
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__drv_sameIRQL
+static VOID
+PollerInstanceDpc(
+ IN PKDPC Dpc,
+ IN PVOID Context,
+ IN PVOID Argument1,
+ IN PVOID Argument2
+ )
+{
+ PXENVIF_POLLER_INSTANCE Instance = Context;
+ PXENVIF_POLLER Poller;
+ PXENVIF_FRONTEND Frontend;
+ BOOLEAN NeedReceiverPoll;
+ BOOLEAN NeedTransmitterPoll;
+
+ UNREFERENCED_PARAMETER(Dpc);
+ UNREFERENCED_PARAMETER(Argument1);
+ UNREFERENCED_PARAMETER(Argument2);
+
+ ASSERT(Instance != NULL);
+
+ Poller = Instance->Poller;
+ Frontend = Poller->Frontend;
+
+ NeedReceiverPoll = FALSE;
+ NeedTransmitterPoll = FALSE;
+
+ for (;;) {
+ if (InterlockedBitTestAndReset(&Instance->Pending,
+ XENVIF_POLLER_EVENT_RECEIVE) != 0)
+ NeedReceiverPoll = TRUE;
+
+ if (InterlockedBitTestAndReset(&Instance->Pending,
+ XENVIF_POLLER_EVENT_TRANSMIT) != 0)
+ NeedTransmitterPoll = TRUE;
+
+ if (!NeedReceiverPoll && !NeedTransmitterPoll)
+ break;
+
+ if (NeedReceiverPoll)
+ {
+ BOOLEAN Retry = ReceiverPoll(FrontendGetReceiver(Frontend),
+ Instance->Index);
+
+ if (!Retry) {
+ NeedReceiverPoll = FALSE;
+ PollerInstanceUnmask(Instance);
+ }
+ }
+
+ if (NeedTransmitterPoll)
+ {
+ BOOLEAN Retry = TransmitterPoll(FrontendGetTransmitter(Frontend),
+ Instance->Index);
+
+ if (!Retry) {
+ NeedTransmitterPoll = FALSE;
+ PollerInstanceUnmask(Instance);
+ }
+ }
+
+ if (PollerInstanceDpcTimeout(Instance)) {
+ PollerInstanceDefer(Instance);
+ break;
+ }
+ }
+}
+
+static NTSTATUS
+PollerInstanceInitialize(
+ IN PXENVIF_POLLER Poller,
+ IN LONG Index,
+ OUT PXENVIF_POLLER_INSTANCE *Instance
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ LONG Type;
+ NTSTATUS status;
+
+ Frontend = Poller->Frontend;
+
+ *Instance = __PollerAllocate(sizeof (XENVIF_POLLER_INSTANCE));
+
+ status = STATUS_NO_MEMORY;
+ if (*Instance == NULL)
+ goto fail1;
+
+ (*Instance)->Poller = Poller;
+ (*Instance)->Index = Index;
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel;
+
+ status = PollerChannelInitialize(*Instance, Type, &Channel);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+ (*Instance)->Channel[Type] = Channel;
+ }
+
+ (*Instance)->Path = FrontendFormatPath(Frontend, Index);
+ if ((*Instance)->Path == NULL)
+ goto fail3;
+
+ KeInitializeSpinLock(&(*Instance)->Lock);
+
+ KeInitializeDpc(&(*Instance)->Dpc, PollerInstanceDpc, *Instance);
+ KeInitializeTimer(&(*Instance)->Timer);
+ KeInitializeDpc(&(*Instance)->TimerDpc, PollerInstanceDpc, *Instance);
+
+ return STATUS_SUCCESS;
+
+fail3:
+ Error("fail3\n");
+
+ Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
+
+fail2:
+ Error("fail2\n");
+
+ while (--Type >= 0)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = (*Instance)->Channel[Type];
+
+ (*Instance)->Channel[Type] = NULL;
+ PollerChannelTeardown(Channel);
+ }
+
+ (*Instance)->Index = 0;
+ (*Instance)->Poller = NULL;
+
+ ASSERT(IsZeroMemory(*Instance, sizeof (XENVIF_POLLER_INSTANCE)));
+ __PollerFree(*Instance);
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static NTSTATUS
+PollerInstanceConnect(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ PROCESSOR_NUMBER ProcNumber;
+ LONG Type;
+ NTSTATUS status;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ status = KeGetProcessorNumberFromIndex(Instance->Index, &ProcNumber);
+ ASSERT(NT_SUCCESS(status));
+
+ KeSetTargetProcessorDpcEx(&Instance->Dpc, &ProcNumber);
+ KeSetTargetProcessorDpcEx(&Instance->TimerDpc, &ProcNumber);
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ status = PollerChannelConnect(Channel);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+ }
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ while (--Type >= 0)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ PollerChannelDisconnect(Channel);
+ }
+
+ return status;
+}
+
+static NTSTATUS
+PollerInstanceStoreWrite(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN PXENBUS_STORE_TRANSACTION Transaction
+ )
+{
+ ULONG Type;
+ NTSTATUS status;
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ status = PollerChannelStoreWrite(Channel, Transaction);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+ }
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
-static FORCEINLINE PVOID
-__PollerAllocate(
- IN ULONG Length
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static NTSTATUS
+PollerInstanceEnable(
+ IN PXENVIF_POLLER_INSTANCE Instance
)
{
- return __AllocatePoolWithTag(NonPagedPool, Length, XENVIF_POLLER_TAG);
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending,
+ XENVIF_POLLER_EVENT_TRANSMIT);
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+ Instance->Enabled = TRUE;
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+
+ (VOID) KeInsertQueueDpc(&Instance->Dpc, NULL, NULL);
+
+ return STATUS_SUCCESS;
}
-static FORCEINLINE VOID
-__PollerFree(
- IN PVOID Buffer
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static NTSTATUS
+PollerInstanceSend(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN XENVIF_POLLER_EVENT_TYPE Event
)
{
- __FreePoolWithTag(Buffer, XENVIF_POLLER_TAG);
+ PXENVIF_POLLER Poller;
+ XENVIF_POLLER_CHANNEL_TYPE Type;
+ PXENVIF_POLLER_CHANNEL Channel;
+ NTSTATUS status;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Poller = Instance->Poller;
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+
+ Type = XENVIF_POLLER_CHANNEL_INVALID;
+
+ if (Instance->Enabled) {
+ if (!__PollerIsSplit(Poller)) {
+ Type = XENVIF_POLLER_CHANNEL_COMBINED;
+ } else {
+ switch (Event) {
+ case XENVIF_POLLER_EVENT_RECEIVE:
+ Type = XENVIF_POLLER_CHANNEL_RECEIVER;
+ break;
+
+ case XENVIF_POLLER_EVENT_TRANSMIT:
+ Type = XENVIF_POLLER_CHANNEL_TRANSMITTER;
+ break;
+
+ default:
+ ASSERT(FALSE);
+ break;
+ }
+ }
+ }
+
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+
+ status = STATUS_UNSUCCESSFUL;
+ if (Type == XENVIF_POLLER_CHANNEL_INVALID)
+ goto fail1;
+
+ Channel = Instance->Channel[Type];
+
+ PollerChannelSend(Channel);
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static NTSTATUS
+PollerInstanceTrigger(
+ IN PXENVIF_POLLER_INSTANCE Instance,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ )
+{
+ NTSTATUS status;
+
+ status = STATUS_INVALID_PARAMETER;
+ if (Event >= XENVIF_POLLER_EVENT_TYPE_COUNT)
+ goto fail1;
+
+ (VOID) InterlockedBitTestAndSet(&Instance->Pending, Event);
+
+ if (KeInsertQueueDpc(&Instance->Dpc, NULL, NULL))
+ Instance->Dpcs++;
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+static VOID
+PollerInstanceDebugCallback(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ PXENVIF_POLLER Poller;
+ ULONG Type;
+
+ Poller = Instance->Poller;
+
+ XENBUS_DEBUG(Printf,
+ &Poller->DebugInterface,
+ "[%d]: Dpcs = %lu\n",
+ Instance->Index,
+ Instance->Dpcs);
+
+ for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ PollerChannelDebugCallback(Channel);
+ }
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static VOID
+PollerInstanceDisable(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
+ Instance->Enabled = FALSE;
+ KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
+
+ //
+ // No new timers can be scheduled once Enabled goes to FALSE.
+ // Cancel any existing ones.
+ //
+ (VOID) KeCancelTimer(&Instance->Timer);
+}
+
+__drv_requiresIRQL(DISPATCH_LEVEL)
+static VOID
+PollerInstanceDisconnect(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ LONG Type;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Instance->Dpcs = 0;
+
+ Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
+
+ while (--Type >= 0)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ PollerChannelDisconnect(Channel);
+ }
+}
+
+static VOID
+PollerInstanceTeardown(
+ IN PXENVIF_POLLER_INSTANCE Instance
+ )
+{
+ PXENVIF_POLLER Poller;
+ PXENVIF_FRONTEND Frontend;
+ LONG Type;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
+ KeFlushQueuedDpcs();
+
+ Poller = Instance->Poller;
+ Frontend = Poller->Frontend;
+
+ RtlZeroMemory(&Instance->TimerDpc, sizeof (KDPC));
+ RtlZeroMemory(&Instance->Timer, sizeof (KTIMER));
+ RtlZeroMemory(&Instance->Dpc, sizeof (KDPC));
+
+ RtlZeroMemory(&Instance->Lock, sizeof (KSPIN_LOCK));
+
+ FrontendFreePath(Frontend, Instance->Path);
+ Instance->Path = NULL;
+
+ Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
+
+ while (--Type >= 0)
+ {
+ PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
+
+ Instance->Channel[Type] = NULL;
+ PollerChannelTeardown(Channel);
+ }
+
+ Instance->Index = 0;
+ Instance->Poller = NULL;
+
+ ASSERT(IsZeroMemory(Instance, sizeof (XENVIF_POLLER_INSTANCE)));
+ __PollerFree(Instance);
}
static VOID
PollerDebugCallback(
- IN PVOID Argument,
- IN BOOLEAN Crashing
+ IN PVOID Argument,
+ IN BOOLEAN Crashing
)
{
- UNREFERENCED_PARAMETER(Argument);
+ PXENVIF_POLLER Poller = Argument;
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ ULONG Index;
+
UNREFERENCED_PARAMETER(Crashing);
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ for (Index = 0; Index < NumQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDebugCallback(Instance);
+ }
+}
+
+static VOID
+PollerSetSplit(
+ IN PXENVIF_POLLER Poller
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ PCHAR Buffer;
+ NTSTATUS status;
+
+ Frontend = Poller->Frontend;
+
+ status = XENBUS_STORE(Read,
+ &Poller->StoreInterface,
+ NULL,
+ FrontendGetBackendPath(Frontend),
+ "feature-split-event-channels",
+ &Buffer);
+ if (NT_SUCCESS(status)) {
+ Poller->Split = (BOOLEAN)strtol(Buffer, NULL, 2);
+
+ XENBUS_STORE(Free,
+ &Poller->StoreInterface,
+ Buffer);
+ } else {
+ Poller->Split = FALSE;
+ }
+
+ Info("%s: %s\n", FrontendGetPath(Frontend),
+ (Poller->Split) ? "TRUE" : "FALSE");
}
NTSTATUS
OUT PXENVIF_POLLER *Poller
)
{
+ LONG MaxQueues;
+ LONG Index;
NTSTATUS status;
*Poller = __PollerAllocate(sizeof (XENVIF_POLLER));
if (*Poller == NULL)
goto fail1;
+ FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
+ &(*Poller)->EvtchnInterface);
+
+ FdoGetStoreInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
+ &(*Poller)->StoreInterface);
+
FdoGetDebugInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
&(*Poller)->DebugInterface);
(*Poller)->Frontend = Frontend;
+ MaxQueues = FrontendGetMaxQueues(Frontend);
+ (*Poller)->Instance = __PollerAllocate(sizeof (PXENVIF_POLLER_INSTANCE) *
+ MaxQueues);
+
+ status = STATUS_NO_MEMORY;
+ if ((*Poller)->Instance == NULL)
+ goto fail2;
+
+ for (Index = 0; Index < MaxQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance;
+
+ status = PollerInstanceInitialize(*Poller, Index, &Instance);
+ if (!NT_SUCCESS(status))
+ goto fail3;
+
+ (*Poller)->Instance[Index] = Instance;
+ }
+
return STATUS_SUCCESS;
+fail3:
+ Error("fail3\n");
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = (*Poller)->Instance[Index];
+
+ (*Poller)->Instance[Index] = NULL;
+ PollerInstanceTeardown(Instance);
+ }
+
+ ASSERT(IsZeroMemory((*Poller)->Instance,
+ sizeof (PXENVIF_POLLER_INSTANCE) * MaxQueues));
+ __PollerFree((*Poller)->Instance);
+ (*Poller)->Instance = NULL;
+
+fail2:
+ Error("fail2\n");
+
+ (*Poller)->Frontend = NULL;
+
+ RtlZeroMemory(&(*Poller)->DebugInterface,
+ sizeof (XENBUS_DEBUG_INTERFACE));
+
+ RtlZeroMemory(&(*Poller)->StoreInterface,
+ sizeof (XENBUS_STORE_INTERFACE));
+
+ RtlZeroMemory(&(*Poller)->EvtchnInterface,
+ sizeof (XENBUS_EVTCHN_INTERFACE));
+
fail1:
Error("fail1 (%08x)\n", status);
IN PXENVIF_POLLER Poller
)
{
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
NTSTATUS status;
Trace("====>\n");
- status = XENBUS_DEBUG(Acquire, &Poller->DebugInterface);
+ Frontend = Poller->Frontend;
+
+ status = XENBUS_EVTCHN(Acquire, &Poller->EvtchnInterface);
if (!NT_SUCCESS(status))
goto fail1;
+ status = XENBUS_STORE(Acquire, &Poller->StoreInterface);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+ status = XENBUS_DEBUG(Acquire, &Poller->DebugInterface);
+ if (!NT_SUCCESS(status))
+ goto fail3;
+
+ PollerSetSplit(Poller);
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ for (Index = 0; Index < NumQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ status = PollerInstanceConnect(Instance);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+ }
+
status = XENBUS_DEBUG(Register,
&Poller->DebugInterface,
__MODULE__ "|POLLER",
Poller,
&Poller->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail2;
+ goto fail5;
Trace("<====\n");
return STATUS_SUCCESS;
+fail5:
+ Error("fail5\n");
+
+ Index = NumQueues;
+
+fail4:
+ Error("fail4\n");
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDisconnect(Instance);
+ }
+
+ Poller->Split = FALSE;
+
+ XENBUS_DEBUG(Release, &Poller->DebugInterface);
+
+fail3:
+ Error("fail3\n");
+
+ XENBUS_STORE(Release, &Poller->StoreInterface);
+
fail2:
Error("fail2\n");
- XENBUS_DEBUG(Release, &Poller->DebugInterface);
+ XENBUS_EVTCHN(Release, &Poller->EvtchnInterface);
fail1:
Error("fail1 (%08x)\n", status);
IN PXENBUS_STORE_TRANSACTION Transaction
)
{
- UNREFERENCED_PARAMETER(Poller);
- UNREFERENCED_PARAMETER(Transaction);
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
+ NTSTATUS status;
+
+ Trace("====>\n");
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ for (Index = 0; Index < NumQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
- Trace("<===>\n");
+ status = PollerInstanceStoreWrite(Instance, Transaction);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+ }
+
+ Trace("<====\n");
return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
}
NTSTATUS
PollerEnable(
- IN PXENVIF_POLLER Poller
+ IN PXENVIF_POLLER Poller
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
+ NTSTATUS status;
+
+ Trace("====>\n");
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ for (Index = 0; Index < NumQueues; Index++) {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ status = PollerInstanceEnable(Instance);
+ if (!NT_SUCCESS(status))
+ goto fail1;
+ }
+
+ Trace("<====\n");
+
+ return STATUS_SUCCESS;
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDisable(Instance);
+ }
+
+ return status;
+}
+
+NTSTATUS
+PollerSend(
+ IN PXENVIF_POLLER Poller,
+ IN ULONG Index,
+ IN XENVIF_POLLER_EVENT_TYPE Event
)
{
- UNREFERENCED_PARAMETER(Poller);
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ PXENVIF_POLLER_INSTANCE Instance;
+ NTSTATUS status;
- Trace("<===>\n");
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ status = STATUS_INVALID_PARAMETER;
+ if (Index >= NumQueues)
+ goto fail1;
+
+ Instance = Poller->Instance[Index];
+
+ status = PollerInstanceSend(Instance, Event);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+ return STATUS_SUCCESS;
+
+fail2:
+ Error("fail2\n");
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
+}
+
+NTSTATUS
+PollerTrigger(
+ IN PXENVIF_POLLER Poller,
+ IN ULONG Index,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ PXENVIF_POLLER_INSTANCE Instance;
+ NTSTATUS status;
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+
+ status = STATUS_INVALID_PARAMETER;
+ if (Index >= NumQueues)
+ goto fail1;
+
+ Instance = Poller->Instance[Index];
+
+ status = PollerInstanceTrigger(Instance, Event);
+ if (!NT_SUCCESS(status))
+ goto fail2;
return STATUS_SUCCESS;
+
+fail2:
+ Error("fail2\n");
+
+fail1:
+ Error("fail1 (%08x)\n", status);
+
+ return status;
}
VOID
PollerDisable(
- IN PXENVIF_POLLER Poller
+ IN PXENVIF_POLLER Poller
)
{
- UNREFERENCED_PARAMETER(Poller);
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
- Trace("<===>\n");
+ Trace("====>\n");
+
+ Frontend = Poller->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+ Index = NumQueues;
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDisable(Instance);
+ }
+
+ Trace("<====\n");
}
VOID
IN PXENVIF_POLLER Poller
)
{
+ PXENVIF_FRONTEND Frontend;
+ LONG NumQueues;
+ LONG Index;
+
Trace("====>\n");
+ Frontend = Poller->Frontend;
+
XENBUS_DEBUG(Deregister,
&Poller->DebugInterface,
Poller->DebugCallback);
Poller->DebugCallback = NULL;
+ NumQueues = FrontendGetNumQueues(Frontend);
+ Index = NumQueues;
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ PollerInstanceDisconnect(Instance);
+ }
+
+ Poller->Split = FALSE;
+
XENBUS_DEBUG(Release, &Poller->DebugInterface);
+ XENBUS_STORE(Release, &Poller->StoreInterface);
+
+ XENBUS_EVTCHN(Release, &Poller->EvtchnInterface);
+
Trace("<====\n");
}
IN PXENVIF_POLLER Poller
)
{
+ PXENVIF_FRONTEND Frontend;
+ LONG MaxQueues;
+ LONG Index;
+
ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
+ Frontend = Poller->Frontend;
+
+ MaxQueues = FrontendGetMaxQueues(Frontend);
+ Index = MaxQueues;
+
+ while (--Index >= 0)
+ {
+ PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
+
+ Poller->Instance[Index] = NULL;
+ PollerInstanceTeardown(Instance);
+ }
+
+ ASSERT(IsZeroMemory(Poller->Instance,
+ sizeof (PXENVIF_POLLER_INSTANCE) * MaxQueues));
+ __PollerFree(Poller->Instance);
+ Poller->Instance = NULL;
+
+ Poller->Frontend = NULL;
+
RtlZeroMemory(&Poller->DebugInterface,
sizeof (XENBUS_DEBUG_INTERFACE));
+ RtlZeroMemory(&Poller->StoreInterface,
+ sizeof (XENBUS_STORE_INTERFACE));
+
+ RtlZeroMemory(&Poller->EvtchnInterface,
+ sizeof (XENBUS_EVTCHN_INTERFACE));
+
ASSERT(IsZeroMemory(Poller, sizeof (XENVIF_POLLER)));
__PollerFree(Poller);
}
typedef struct _XENVIF_POLLER XENVIF_POLLER, *PXENVIF_POLLER;
+typedef enum _XENVIF_POLLER_EVENT_TYPE {
+ XENVIF_POLLER_EVENT_RECEIVE,
+ XENVIF_POLLER_EVENT_TRANSMIT,
+ XENVIF_POLLER_EVENT_TYPE_COUNT
+} XENVIF_POLLER_EVENT_TYPE, *PXENVIF_POLLER_EVENT_TYPE;
+
extern NTSTATUS
PollerInitialize(
IN PXENVIF_FRONTEND Frontend,
- OUT PXENVIF_POLLER *Poller
+ OUT PXENVIF_POLLER *Poller
);
extern NTSTATUS
extern NTSTATUS
PollerStoreWrite(
- IN PXENVIF_POLLER Poller,
+ IN PXENVIF_POLLER Poller,
IN PXENBUS_STORE_TRANSACTION Transaction
);
IN PXENVIF_POLLER Poller
);
+extern NTSTATUS
+PollerSend(
+ IN PXENVIF_POLLER Poller,
+ IN ULONG Index,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ );
+
+extern NTSTATUS
+PollerTrigger(
+ IN PXENVIF_POLLER Poller,
+ IN ULONG Index,
+ IN XENVIF_POLLER_EVENT_TYPE Event
+ );
+
extern VOID
PollerDisable(
IN PXENVIF_POLLER Poller
#include <store_interface.h>
#include <cache_interface.h>
#include <gnttab_interface.h>
-#include <evtchn_interface.h>
#include "pdo.h"
#include "registry.h"
netif_rx_front_ring_t Front;
netif_rx_sring_t *Shared;
PXENBUS_GNTTAB_ENTRY Entry;
- PXENBUS_EVTCHN_CHANNEL Channel;
- KDPC Dpc;
- ULONG Dpcs;
- KTIMER Timer;
- KDPC TimerDpc;
- ULONG Events;
PXENVIF_RECEIVER_FRAGMENT Pending[XENVIF_RECEIVER_MAXIMUM_FRAGMENT_ID + 1];
ULONG RequestsPosted;
ULONG RequestsPushed;
PXENVIF_FRONTEND Frontend;
XENBUS_CACHE_INTERFACE CacheInterface;
XENBUS_GNTTAB_INTERFACE GnttabInterface;
- XENBUS_EVTCHN_INTERFACE EvtchnInterface;
PXENVIF_RECEIVER_RING *Ring;
LONG Loaned;
LONG Returned;
return Ring->Stopped;
}
-static FORCEINLINE VOID
-__ReceiverRingTrigger(
- IN PXENVIF_RECEIVER_RING Ring,
- IN BOOLEAN Locked
- )
-{
- PXENVIF_RECEIVER Receiver;
-
- Receiver = Ring->Receiver;
-
- if (!Locked)
- __ReceiverRingAcquireLock(Ring);
-
- if (Ring->Connected)
- (VOID) XENBUS_EVTCHN(Trigger,
- &Receiver->EvtchnInterface,
- Ring->Channel);
-
- if (!Locked)
- __ReceiverRingReleaseLock(Ring);
-}
-
-static FORCEINLINE VOID
-__ReceiverRingSend(
- IN PXENVIF_RECEIVER_RING Ring,
- IN BOOLEAN Locked
- )
-{
- PXENVIF_RECEIVER Receiver;
-
- Receiver = Ring->Receiver;
-
- if (!Locked)
- __ReceiverRingAcquireLock(Ring);
-
- if (Ring->Connected)
- (VOID) XENBUS_EVTCHN(Send,
- &Receiver->EvtchnInterface,
- Ring->Channel);
-
- if (!Locked)
- __ReceiverRingReleaseLock(Ring);
-}
-
static FORCEINLINE VOID
__ReceiverRingReturnPacket(
IN PXENVIF_RECEIVER_RING Ring,
IN BOOLEAN Locked
)
{
+ PXENVIF_RECEIVER Receiver;
+ PXENVIF_FRONTEND Frontend;
PMDL Mdl;
+ Receiver = Ring->Receiver;
+ Frontend = Receiver->Frontend;
+
Mdl = &Packet->Mdl;
while (Mdl != NULL) {
if (__ReceiverRingIsStopped(Ring)) {
__ReceiverRingStart(Ring);
- __ReceiverRingTrigger(Ring, TRUE);
+ PollerTrigger(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_RECEIVE);
}
if (!Locked)
#pragma warning (pop)
- if (Notify)
- __ReceiverRingSend(Ring, TRUE);
+ if (Notify) {
+ PXENVIF_RECEIVER Receiver;
+ PXENVIF_FRONTEND Frontend;
+
+ Receiver = Ring->Receiver;
+ Frontend = Receiver->Frontend;
+
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ }
Ring->RequestsPushed = Ring->RequestsPosted;
}
Ring->RequestsPosted,
Ring->RequestsPushed,
Ring->ResponsesProcessed);
-
- // Dump event channel
- XENBUS_DEBUG(Printf,
- &Receiver->DebugInterface,
- "[%s]: Events = %lu Dpcs = %lu\n",
- FrontendIsSplit(Frontend) ? "RX" : "COMBINED",
- Ring->Events,
- Ring->Dpcs);
}
static DECLSPEC_NOINLINE BOOLEAN
#undef XENVIF_RECEIVER_BATCH
}
-static FORCEINLINE VOID
-__ReceiverRingUnmask(
- IN PXENVIF_RECEIVER_RING Ring
- )
-{
- PXENVIF_RECEIVER Receiver;
-
- if (!Ring->Connected)
- return;
-
- Receiver = Ring->Receiver;
-
- XENBUS_EVTCHN(Unmask,
- &Receiver->EvtchnInterface,
- Ring->Channel,
- FALSE);
-}
-
-static FORCEINLINE BOOLEAN
-__ReceiverRingDpcTimeout(
- IN PXENVIF_RECEIVER_RING Ring
- )
-{
- KDPC_WATCHDOG_INFORMATION Watchdog;
- NTSTATUS status;
-
- UNREFERENCED_PARAMETER(Ring);
-
- RtlZeroMemory(&Watchdog, sizeof (Watchdog));
-
- status = KeQueryDpcWatchdogInformation(&Watchdog);
- ASSERT(NT_SUCCESS(status));
-
- if (Watchdog.DpcTimeLimit == 0 ||
- Watchdog.DpcWatchdogLimit == 0)
- return FALSE;
-
- if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) &&
- Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2))
- return FALSE;
-
- return TRUE;
-}
-
#define TIME_US(_us) ((_us) * 10)
#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
#define TIME_S(_s) (TIME_MS((_s) * 1000))
#define TIME_RELATIVE(_t) (-(_t))
-__drv_functionClass(KDEFERRED_ROUTINE)
-__drv_maxIRQL(DISPATCH_LEVEL)
-__drv_minIRQL(DISPATCH_LEVEL)
-__drv_requiresIRQL(DISPATCH_LEVEL)
-__drv_sameIRQL
-static VOID
-ReceiverRingDpc(
- IN PKDPC Dpc,
- IN PVOID Context,
- IN PVOID Argument1,
- IN PVOID Argument2
- )
-{
- PXENVIF_RECEIVER_RING Ring = Context;
-
- UNREFERENCED_PARAMETER(Dpc);
- UNREFERENCED_PARAMETER(Argument1);
- UNREFERENCED_PARAMETER(Argument2);
-
- ASSERT(Ring != NULL);
-
- for (;;) {
- BOOLEAN Retry;
-
- __ReceiverRingAcquireLock(Ring);
- Retry = ReceiverRingPoll(Ring);
- __ReceiverRingReleaseLock(Ring);
-
- if (!Retry) {
- __ReceiverRingUnmask(Ring);
- break;
- }
-
- if (__ReceiverRingDpcTimeout(Ring)) {
- LARGE_INTEGER Delay;
-
- Delay.QuadPart = TIME_RELATIVE(TIME_US(100));
-
- KeSetTimer(&Ring->Timer, Delay, &Ring->TimerDpc);
- break;
- }
- }
-}
-
-KSERVICE_ROUTINE ReceiverRingEvtchnCallback;
-
-BOOLEAN
-ReceiverRingEvtchnCallback(
- IN PKINTERRUPT InterruptObject,
- IN PVOID Argument
- )
-{
- PXENVIF_RECEIVER_RING Ring = Argument;
- PXENVIF_RECEIVER Receiver;
- PXENVIF_FRONTEND Frontend;
-
- UNREFERENCED_PARAMETER(InterruptObject);
-
- ASSERT(Ring != NULL);
-
- Ring->Events++;
-
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- Ring->Dpcs++;
-
- Receiver = Ring->Receiver;
- Frontend = Receiver->Frontend;
-
- if (!FrontendIsSplit(Frontend))
- TransmitterNotify(FrontendGetTransmitter(Frontend),
- Ring->Index);
-
- return TRUE;
-}
-
#define XENVIF_RECEIVER_WATCHDOG_PERIOD 30
static NTSTATUS
if (Ring->Shared->rsp_prod != rsp_prod &&
Ring->Front.rsp_cons == rsp_cons) {
PXENVIF_RECEIVER Receiver;
+ PXENVIF_FRONTEND Frontend;
Receiver = Ring->Receiver;
+ Frontend = Receiver->Frontend;
XENBUS_DEBUG(Trigger,
&Receiver->DebugInterface,
Ring->DebugCallback);
// Try to move things along
- __ReceiverRingTrigger(Ring, TRUE);
- __ReceiverRingSend(Ring, TRUE);
+ PollerTrigger(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_RECEIVE);
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_RECEIVE);
}
KeMemoryBarrier();
InitializeListHead(&(*Ring)->PacketList);
- KeInitializeDpc(&(*Ring)->Dpc, ReceiverRingDpc, *Ring);
- KeInitializeTimer(&(*Ring)->Timer);
- KeInitializeDpc(&(*Ring)->TimerDpc, ReceiverRingDpc, *Ring);
-
status = RtlStringCbPrintfA(Name,
sizeof (Name),
"%s_receiver_packet",
fail3:
Error("fail3\n");
- RtlZeroMemory(&(*Ring)->TimerDpc, sizeof (KDPC));
- RtlZeroMemory(&(*Ring)->Timer, sizeof (KTIMER));
- RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC));
-
RtlZeroMemory(&(*Ring)->PacketList, sizeof (LIST_ENTRY));
FrontendFreePath(Frontend, (*Ring)->Path);
PFN_NUMBER Pfn;
CHAR Name[MAXNAMELEN];
ULONG Index;
- PROCESSOR_NUMBER ProcNumber;
NTSTATUS status;
Receiver = Ring->Receiver;
ASSERT(!Ring->Connected);
- Ring->Channel = XENBUS_EVTCHN(Open,
- &Receiver->EvtchnInterface,
- XENBUS_EVTCHN_TYPE_UNBOUND,
- ReceiverRingEvtchnCallback,
- Ring,
- FrontendGetBackendDomain(Frontend),
- TRUE);
-
- status = STATUS_UNSUCCESSFUL;
- if (Ring->Channel == NULL)
- goto fail6;
-
- status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber);
- ASSERT(NT_SUCCESS(status));
-
- KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber);
- KeSetTargetProcessorDpcEx(&Ring->TimerDpc, &ProcNumber);
-
- (VOID) XENBUS_EVTCHN(Bind,
- &Receiver->EvtchnInterface,
- Ring->Channel,
- ProcNumber.Group,
- ProcNumber.Number);
-
- XENBUS_EVTCHN(Unmask,
- &Receiver->EvtchnInterface,
- Ring->Channel,
- FALSE);
-
Ring->Connected = TRUE;
status = XENBUS_DEBUG(Register,
Ring,
&Ring->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail7;
+ goto fail6;
return STATUS_SUCCESS;
-fail7:
- Error("fail7\n");
-
- Ring->Connected = FALSE;
-
- XENBUS_EVTCHN(Close,
- &Receiver->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- Ring->Events = 0;
-
fail6:
Error("fail6\n");
+ Ring->Connected = FALSE;
+
fail5:
Error("fail5\n");
{
PXENVIF_RECEIVER Receiver;
PXENVIF_FRONTEND Frontend;
- ULONG Port;
PCHAR Path;
NTSTATUS status;
if (!NT_SUCCESS(status))
goto fail1;
- Port = XENBUS_EVTCHN(GetPort,
- &Receiver->EvtchnInterface,
- Ring->Channel);
-
- status = XENBUS_STORE(Printf,
- &Receiver->StoreInterface,
- Transaction,
- Path,
- FrontendIsSplit(Frontend) ? "event-channel-rx" : "event-channel",
- "%u",
- Port);
- if (!NT_SUCCESS(status))
- goto fail2;
-
return STATUS_SUCCESS;
-fail2:
- Error("fail2\n");
-
fail1:
Error("fail1 (%08x)\n", status);
Ring->Enabled = TRUE;
- (VOID) KeInsertQueueDpc(&Ring->Dpc, NULL, NULL);
-
__ReceiverRingReleaseLock(Ring);
Info("%s[%u]: <====\n",
__ReceiverRingReleaseLock(Ring);
- //
- // No new timers can be scheduled once Enabled goes to FALSE.
- // Cancel any existing ones.
- //
- (VOID) KeCancelTimer(&Ring->Timer);
-
Info("%s[%u]: <====\n",
FrontendGetPath(Frontend),
Ring->Index);
ASSERT(Ring->Connected);
Ring->Connected = FALSE;
- XENBUS_EVTCHN(Close,
- &Receiver->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- Ring->Events = 0;
- Ring->Dpcs = 0;
-
ASSERT3U(Ring->ResponsesProcessed, ==, Ring->RequestsPushed);
ASSERT3U(Ring->RequestsPushed, ==, Ring->RequestsPosted);
Frontend = Receiver->Frontend;
RtlZeroMemory(&Ring->Hash, sizeof (XENVIF_RECEIVER_HASH));
- RtlZeroMemory(&Ring->TimerDpc, sizeof (KDPC));
- RtlZeroMemory(&Ring->Timer, sizeof (KTIMER));
- RtlZeroMemory(&Ring->Dpc, sizeof (KDPC));
Ring->BackfillSize = 0;
Ring->OffloadOptions.Value = 0;
FdoGetGnttabInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
&(*Receiver)->GnttabInterface);
- FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
- &(*Receiver)->EvtchnInterface);
-
(*Receiver)->Frontend = Frontend;
status = XENBUS_CACHE(Acquire, &(*Receiver)->CacheInterface);
(*Receiver)->Frontend = NULL;
- RtlZeroMemory(&(*Receiver)->EvtchnInterface,
- sizeof (XENBUS_EVTCHN_INTERFACE));
-
RtlZeroMemory(&(*Receiver)->GnttabInterface,
sizeof (XENBUS_GNTTAB_INTERFACE));
if (!NT_SUCCESS(status))
goto fail2;
- status = XENBUS_EVTCHN(Acquire, &Receiver->EvtchnInterface);
- if (!NT_SUCCESS(status))
- goto fail3;
-
status = XENBUS_GNTTAB(Acquire, &Receiver->GnttabInterface);
if (!NT_SUCCESS(status))
- goto fail4;
+ goto fail3;
Index = 0;
while (Index < (LONG)FrontendGetNumQueues(Frontend)) {
status = __ReceiverRingConnect(Ring);
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail4;
Index++;
}
Receiver,
&Receiver->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail6;
+ goto fail5;
Trace("<====\n");
return STATUS_SUCCESS;
-fail6:
- Error("fail6\n");
+fail5:
+ Error("fail5\n");
Index = FrontendGetNumQueues(Frontend);
-fail5:
- Error("fail5\n");
+fail4:
+ Error("fail4\n");
while (--Index >= 0) {
PXENVIF_RECEIVER_RING Ring = Receiver->Ring[Index];
XENBUS_GNTTAB(Release, &Receiver->GnttabInterface);
-fail4:
- Error("fail4\n");
-
- XENBUS_EVTCHN(Release, &Receiver->EvtchnInterface);
-
fail3:
Error("fail3\n");
return status;
}
+BOOLEAN
+ReceiverPoll(
+ IN PXENVIF_RECEIVER Receiver,
+ IN ULONG Index
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ PXENVIF_RECEIVER_RING Ring;
+ BOOLEAN Retry;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Frontend = Receiver->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+ if (Index >= NumQueues)
+ return FALSE;
+
+ Ring = Receiver->Ring[Index];
+
+ __ReceiverRingAcquireLock(Ring);
+ Retry = ReceiverRingPoll(Ring);
+ __ReceiverRingReleaseLock(Ring);
+
+ return Retry;
+}
+
VOID
ReceiverDisable(
IN PXENVIF_RECEIVER Receiver
XENBUS_GNTTAB(Release, &Receiver->GnttabInterface);
- XENBUS_EVTCHN(Release, &Receiver->EvtchnInterface);
-
XENBUS_STORE(Release, &Receiver->StoreInterface);
XENBUS_DEBUG(Release, &Receiver->DebugInterface);
Frontend = Receiver->Frontend;
- ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
- KeFlushQueuedDpcs();
-
ASSERT3U(Receiver->Returned, ==, Receiver->Loaned);
Receiver->Loaned = 0;
Receiver->Returned = 0;
Receiver->Frontend = NULL;
- RtlZeroMemory(&Receiver->EvtchnInterface,
- sizeof (XENBUS_EVTCHN_INTERFACE));
-
RtlZeroMemory(&Receiver->GnttabInterface,
sizeof (XENBUS_GNTTAB_INTERFACE));
Trace("%s: <====\n", FrontendGetPath(Frontend));
}
-VOID
-ReceiverTrigger(
- IN PXENVIF_RECEIVER Receiver,
- IN ULONG Index
- )
-{
- PXENVIF_RECEIVER_RING Ring;
-
- Ring = Receiver->Ring[Index];
-
- __ReceiverRingTrigger(Ring, FALSE);
-}
-
-VOID
-ReceiverSend(
- IN PXENVIF_RECEIVER Receiver,
- IN ULONG Index
- )
-{
- PXENVIF_RECEIVER_RING Ring;
-
- Ring = Receiver->Ring[Index];
-
- __ReceiverRingSend(Ring, FALSE);
-}
-
NTSTATUS
ReceiverSetHashAlgorithm(
IN PXENVIF_RECEIVER Receiver,
IN PXENVIF_RECEIVER Receiver
);
+extern BOOLEAN
+ReceiverPoll(
+ IN PXENVIF_RECEIVER Receiver,
+ IN ULONG Index
+ );
+
extern VOID
ReceiverDisable(
IN PXENVIF_RECEIVER Receiver
#include <cache_interface.h>
#include <gnttab_interface.h>
#include <range_set_interface.h>
-#include <evtchn_interface.h>
#include "pdo.h"
#include "frontend.h"
netif_tx_front_ring_t Front;
netif_tx_sring_t *Shared;
PXENBUS_GNTTAB_ENTRY Entry;
- PXENBUS_EVTCHN_CHANNEL Channel;
- KDPC Dpc;
- ULONG Dpcs;
- KTIMER Timer;
- KDPC TimerDpc;
- ULONG Events;
BOOLEAN Connected;
BOOLEAN Enabled;
BOOLEAN Stopped;
XENBUS_CACHE_INTERFACE CacheInterface;
XENBUS_GNTTAB_INTERFACE GnttabInterface;
XENBUS_RANGE_SET_INTERFACE RangeSetInterface;
- XENBUS_EVTCHN_INTERFACE EvtchnInterface;
PXENVIF_TRANSMITTER_RING *Ring;
BOOLEAN MulticastControl;
ULONG DisableIpVersion4Gso;
Ring->PacketsUnprepared,
Ring->PacketsSent,
Ring->PacketsCompleted);
-
- if (FrontendIsSplit(Frontend)) {
- // Dump event channel
- XENBUS_DEBUG(Printf,
- &Transmitter->DebugInterface,
- "Events = %lu Dpcs = %lu\n",
- Ring->Events,
- Ring->Dpcs);
- }
}
static BOOLEAN
}
static FORCEINLINE VOID
-__TransmitterRingTrigger(
- IN PXENVIF_TRANSMITTER_RING Ring
- )
-{
- PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_FRONTEND Frontend;
-
- Transmitter = Ring->Transmitter;
- Frontend = Transmitter->Frontend;
-
- if (!Ring->Connected)
- return;
-
- if (FrontendIsSplit(Frontend)) {
- ASSERT(Ring->Channel != NULL);
-
- (VOID) XENBUS_EVTCHN(Trigger,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
- } else {
- ReceiverTrigger(FrontendGetReceiver(Frontend),
- Ring->Index);
- }
-}
-
-static FORCEINLINE VOID
-__TransmitterRingSend(
+__TransmitterRingPushRequests(
IN PXENVIF_TRANSMITTER_RING Ring
)
{
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
+ BOOLEAN Notify;
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
- if (!Ring->Connected)
- return;
-
- if (FrontendIsSplit(Frontend)) {
- ASSERT(Ring->Channel != NULL);
-
- (VOID) XENBUS_EVTCHN(Send,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
- } else {
- ReceiverSend(FrontendGetReceiver(Frontend),
- Ring->Index);
- }
-}
-
-static FORCEINLINE VOID
-__TransmitterRingPushRequests(
- IN PXENVIF_TRANSMITTER_RING Ring
- )
-{
- BOOLEAN Notify;
-
if (Ring->RequestsPosted == Ring->RequestsPushed)
return;
#pragma warning (pop)
if (Notify)
- __TransmitterRingSend(Ring);
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_TRANSMIT);
Ring->RequestsPushed = Ring->RequestsPosted;
}
__TransmitterRingReleaseLock(Ring);
}
-static FORCEINLINE VOID
-__TransmitterRingUnmask(
- IN PXENVIF_TRANSMITTER_RING Ring
- )
-{
- PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_FRONTEND Frontend;
-
- Transmitter = Ring->Transmitter;
- Frontend = Transmitter->Frontend;
-
- if (!Ring->Connected || !FrontendIsSplit(Frontend))
- return;
-
- XENBUS_EVTCHN(Unmask,
- &Transmitter->EvtchnInterface,
- Ring->Channel,
- FALSE);
-}
-
-static FORCEINLINE BOOLEAN
-__TransmitterRingDpcTimeout(
- IN PXENVIF_TRANSMITTER_RING Ring
- )
-{
- KDPC_WATCHDOG_INFORMATION Watchdog;
- NTSTATUS status;
-
- UNREFERENCED_PARAMETER(Ring);
-
- RtlZeroMemory(&Watchdog, sizeof (Watchdog));
-
- status = KeQueryDpcWatchdogInformation(&Watchdog);
- ASSERT(NT_SUCCESS(status));
-
- if (Watchdog.DpcTimeLimit == 0 ||
- Watchdog.DpcWatchdogLimit == 0)
- return FALSE;
-
- if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) &&
- Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2))
- return FALSE;
-
- return TRUE;
-}
-
#define TIME_US(_us) ((_us) * 10)
#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
#define TIME_S(_s) (TIME_MS((_s) * 1000))
#define TIME_RELATIVE(_t) (-(_t))
-__drv_functionClass(KDEFERRED_ROUTINE)
-__drv_maxIRQL(DISPATCH_LEVEL)
-__drv_minIRQL(DISPATCH_LEVEL)
-__drv_requiresIRQL(DISPATCH_LEVEL)
-__drv_sameIRQL
-static VOID
-TransmitterRingDpc(
- IN PKDPC Dpc,
- IN PVOID Context,
- IN PVOID Argument1,
- IN PVOID Argument2
- )
-{
- PXENVIF_TRANSMITTER_RING Ring = Context;
-
- UNREFERENCED_PARAMETER(Dpc);
- UNREFERENCED_PARAMETER(Argument1);
- UNREFERENCED_PARAMETER(Argument2);
-
- ASSERT(Ring != NULL);
-
- for (;;) {
- BOOLEAN Retry;
-
- __TransmitterRingAcquireLock(Ring);
- Retry = TransmitterRingPoll(Ring);
- __TransmitterRingReleaseLock(Ring);
-
- if (!Retry) {
- __TransmitterRingUnmask(Ring);
- break;
- }
-
- if (__TransmitterRingDpcTimeout(Ring)) {
- LARGE_INTEGER Delay;
-
- Delay.QuadPart = TIME_RELATIVE(TIME_US(100));
-
- KeSetTimer(&Ring->Timer, Delay, &Ring->TimerDpc);
- break;
- }
- }
-}
-
-KSERVICE_ROUTINE TransmitterRingEvtchnCallback;
-
-BOOLEAN
-TransmitterRingEvtchnCallback(
- IN PKINTERRUPT InterruptObject,
- IN PVOID Argument
- )
-{
- PXENVIF_TRANSMITTER_RING Ring = Argument;
- PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_FRONTEND Frontend;
-
- UNREFERENCED_PARAMETER(InterruptObject);
-
- ASSERT(Ring != NULL);
-
- Transmitter = Ring->Transmitter;
- Frontend = Transmitter->Frontend;
-
- ASSERT(FrontendIsSplit(Frontend));
-
- Ring->Events++;
-
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- Ring->Dpcs++;
-
- return TRUE;
-}
-
#define XENVIF_TRANSMITTER_WATCHDOG_PERIOD 30
static NTSTATUS
if (Ring->PacketsQueued == PacketsQueued &&
Ring->PacketsCompleted != PacketsQueued) {
PXENVIF_TRANSMITTER Transmitter;
+ PXENVIF_FRONTEND Frontend;
Transmitter = Ring->Transmitter;
+ Frontend = Transmitter->Frontend;
XENBUS_DEBUG(Trigger,
&Transmitter->DebugInterface,
Ring->DebugCallback);
// Try to move things along
- __TransmitterRingTrigger(Ring);
- __TransmitterRingSend(Ring);
+ PollerTrigger(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_TRANSMIT);
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_TRANSMIT);
}
PacketsQueued = Ring->PacketsQueued;
InitializeListHead(&(*Ring)->RequestQueue);
InitializeListHead(&(*Ring)->PacketComplete);
- KeInitializeDpc(&(*Ring)->Dpc, TransmitterRingDpc, *Ring);
- KeInitializeTimer(&(*Ring)->Timer);
- KeInitializeDpc(&(*Ring)->TimerDpc, TransmitterRingDpc, *Ring);
-
status = RtlStringCbPrintfA(Name,
sizeof (Name),
"%s_transmitter_buffer",
fail3:
Error("fail3\n");
- RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC));
-
RtlZeroMemory(&(*Ring)->PacketComplete, sizeof (LIST_ENTRY));
RtlZeroMemory(&(*Ring)->RequestQueue, sizeof (LIST_ENTRY));
RtlZeroMemory(&(*Ring)->PacketQueue, sizeof (LIST_ENTRY));
PFN_NUMBER Pfn;
CHAR Name[MAXNAMELEN];
ULONG Index;
- PROCESSOR_NUMBER ProcNumber;
NTSTATUS status;
ASSERT(!Ring->Connected);
ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
- if (FrontendIsSplit(Frontend)) {
- Ring->Channel = XENBUS_EVTCHN(Open,
- &Transmitter->EvtchnInterface,
- XENBUS_EVTCHN_TYPE_UNBOUND,
- TransmitterRingEvtchnCallback,
- Ring,
- FrontendGetBackendDomain(Frontend),
- TRUE);
-
- status = STATUS_UNSUCCESSFUL;
- if (Ring->Channel == NULL)
- goto fail6;
-
- status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber);
- ASSERT(NT_SUCCESS(status));
-
- KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber);
- KeSetTargetProcessorDpcEx(&Ring->TimerDpc, &ProcNumber);
-
- (VOID) XENBUS_EVTCHN(Bind,
- &Transmitter->EvtchnInterface,
- Ring->Channel,
- ProcNumber.Group,
- ProcNumber.Number);
-
- XENBUS_EVTCHN(Unmask,
- &Transmitter->EvtchnInterface,
- Ring->Channel,
- FALSE);
- }
-
status = XENBUS_DEBUG(Register,
&Transmitter->DebugInterface,
Name,
Ring,
&Ring->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail7;
+ goto fail6;
Ring->Connected = TRUE;
return STATUS_SUCCESS;
-fail7:
- Error("fail7\n");
-
- XENBUS_EVTCHN(Close,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- Ring->Events = 0;
-
fail6:
Error("fail6\n");
{
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
- ULONG Port;
PCHAR Path;
NTSTATUS status;
if (!NT_SUCCESS(status))
goto fail1;
- if (!FrontendIsSplit(Frontend))
- goto done;
-
- Port = XENBUS_EVTCHN(GetPort,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
-
- status = XENBUS_STORE(Printf,
- &Transmitter->StoreInterface,
- Transaction,
- Path,
- "event-channel-tx",
- "%u",
- Port);
- if (!NT_SUCCESS(status))
- goto fail2;
-
-done:
return STATUS_SUCCESS;
-fail2:
- Error("fail2\n");
-
fail1:
Error("fail1 (%08x)\n", status);
ASSERT(!Ring->Enabled);
Ring->Enabled = TRUE;
- KeInsertQueueDpc(&Ring->Dpc, NULL, NULL);
-
__TransmitterRingReleaseLock(Ring);
Info("%s[%u]: <====\n",
ASSERT(Attempt < 100);
// Try to move things along
- __TransmitterRingSend(Ring);
+ PollerSend(FrontendGetPoller(Frontend),
+ Ring->Index,
+ XENVIF_POLLER_EVENT_TRANSMIT);
(VOID) TransmitterRingPoll(Ring);
if (State != XenbusStateConnected)
__TransmitterRingReleaseLock(Ring);
- //
- // No new timers can be scheduled once Enabled goes to FALSE.
- // Cancel any existing ones.
- //
- (VOID) KeCancelTimer(&Ring->Timer);
-
Info("%s[%u]: <====\n",
FrontendGetPath(Frontend),
Ring->Index);
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
- if (Ring->Channel != NULL) {
- XENBUS_EVTCHN(Close,
- &Transmitter->EvtchnInterface,
- Ring->Channel);
- Ring->Channel = NULL;
-
- Ring->Events = 0;
- }
-
- Ring->Dpcs = 0;
-
ASSERT3U(Ring->ResponsesProcessed, ==, Ring->RequestsPushed);
ASSERT3U(Ring->RequestsPushed, ==, Ring->RequestsPosted);
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
- Ring->Dpcs = 0;
-
- RtlZeroMemory(&Ring->TimerDpc, sizeof (KDPC));
- RtlZeroMemory(&Ring->Timer, sizeof (KTIMER));
- RtlZeroMemory(&Ring->Dpc, sizeof (KDPC));
-
ASSERT3U(Ring->PacketsCompleted, ==, Ring->PacketsSent);
ASSERT3U(Ring->PacketsSent, ==, Ring->PacketsPrepared - Ring->PacketsUnprepared);
ASSERT3U(Ring->PacketsPrepared, ==, Ring->PacketsCopied + Ring->PacketsGranted + Ring->PacketsFaked);
FdoGetGnttabInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
&(*Transmitter)->GnttabInterface);
- FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
- &(*Transmitter)->EvtchnInterface);
-
(*Transmitter)->Frontend = Frontend;
KeInitializeSpinLock(&(*Transmitter)->Lock);
if (!NT_SUCCESS(status))
goto fail2;
- status = XENBUS_EVTCHN(Acquire, &Transmitter->EvtchnInterface);
- if (!NT_SUCCESS(status))
- goto fail3;
-
status = XENBUS_GNTTAB(Acquire, &Transmitter->GnttabInterface);
if (!NT_SUCCESS(status))
- goto fail4;
+ goto fail3;
if (Transmitter->DisableMulticastControl == 0) {
status = XENBUS_STORE(Read,
status = __TransmitterRingConnect(Ring);
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail4;
Index++;
}
Transmitter,
&Transmitter->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail6;
+ goto fail5;
Trace("<====\n");
return STATUS_SUCCESS;
-fail6:
- Error("fail6\n");
+fail5:
+ Error("fail5\n");
Index = FrontendGetNumQueues(Frontend);
-fail5:
- Error("fail5\n");
+fail4:
+ Error("fail4\n");
while (--Index >= 0) {
PXENVIF_TRANSMITTER_RING Ring;
XENBUS_GNTTAB(Release, &Transmitter->GnttabInterface);
-fail4:
- Error("fail4\n");
-
- XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface);
-
fail3:
Error("fail3\n");
return STATUS_SUCCESS;
}
+BOOLEAN
+TransmitterPoll(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN ULONG Index
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ ULONG NumQueues;
+ PXENVIF_TRANSMITTER_RING Ring;
+ BOOLEAN Retry;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ Frontend = Transmitter->Frontend;
+
+ NumQueues = FrontendGetNumQueues(Frontend);
+ if (Index >= NumQueues)
+ return FALSE;
+
+ Ring = Transmitter->Ring[Index];
+
+ __TransmitterRingAcquireLock(Ring);
+ Retry = TransmitterRingPoll(Ring);
+ __TransmitterRingReleaseLock(Ring);
+
+ return Retry;
+}
+
VOID
TransmitterDisable(
IN PXENVIF_TRANSMITTER Transmitter
XENBUS_GNTTAB(Release, &Transmitter->GnttabInterface);
- XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface);
-
XENBUS_STORE(Release, &Transmitter->StoreInterface);
XENBUS_DEBUG(Release, &Transmitter->DebugInterface);
Frontend = Transmitter->Frontend;
ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
- KeFlushQueuedDpcs();
Index = FrontendGetMaxQueues(Frontend);
while (--Index >= 0) {
RtlZeroMemory(&Transmitter->DebugInterface,
sizeof (XENBUS_DEBUG_INTERFACE));
- RtlZeroMemory(&Transmitter->EvtchnInterface,
- sizeof (XENBUS_EVTCHN_INTERFACE));
-
Transmitter->DisableIpVersion4Gso = 0;
Transmitter->DisableIpVersion6Gso = 0;
Transmitter->AlwaysCopy = 0;
*Size = XENVIF_TRANSMITTER_RING_SIZE;
}
-VOID
-TransmitterNotify(
- IN PXENVIF_TRANSMITTER Transmitter,
- IN ULONG Index
- )
-{
- PXENVIF_FRONTEND Frontend;
- PXENVIF_TRANSMITTER_RING Ring;
-
- Frontend = Transmitter->Frontend;
-
- ASSERT(!FrontendIsSplit(Frontend));
-
- Ring = Transmitter->Ring[Index];
-
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- Ring->Dpcs++;
-}
-
VOID
TransmitterQueryOffloadOptions(
IN PXENVIF_TRANSMITTER Transmitter,
IN PXENVIF_TRANSMITTER Transmitter
);
+extern BOOLEAN
+TransmitterPoll(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN ULONG Index
+ );
+
extern VOID
TransmitterDisable(
IN PXENVIF_TRANSMITTER Transmitter
projects / people / pauldu / xenvif.git / commitdiff