#include "tcpip.h"
#include "receiver.h"
#include "transmitter.h"
-#include "poller.h"
#include "link.h"
#include "dbg_print.h"
#include "assert.h"
USHORT BackendDomain;
ULONG MaxQueues;
ULONG NumQueues;
+ BOOLEAN Split;
ULONG DisableToeplitz;
PXENVIF_MAC Mac;
PXENVIF_RECEIVER Receiver;
PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_POLLER Poller;
PXENVIF_CONTROLLER Controller;
XENBUS_DEBUG_INTERFACE DebugInterface;
DEFINE_FRONTEND_GET_FUNCTION(Mac, PXENVIF_MAC)
DEFINE_FRONTEND_GET_FUNCTION(Receiver, PXENVIF_RECEIVER)
DEFINE_FRONTEND_GET_FUNCTION(Transmitter, PXENVIF_TRANSMITTER)
-DEFINE_FRONTEND_GET_FUNCTION(Poller, PXENVIF_POLLER)
DEFINE_FRONTEND_GET_FUNCTION(Controller, PXENVIF_CONTROLLER)
static BOOLEAN
return __FrontendGetNumQueues(Frontend);
}
+static VOID
+FrontendSetSplit(
+ IN PXENVIF_FRONTEND Frontend
+ )
+{
+ PCHAR Buffer;
+ NTSTATUS status;
+
+ status = XENBUS_STORE(Read,
+ &Frontend->StoreInterface,
+ NULL,
+ __FrontendGetBackendPath(Frontend),
+ "feature-split-event-channels",
+ &Buffer);
+ if (NT_SUCCESS(status)) {
+ Frontend->Split = (BOOLEAN)strtol(Buffer, NULL, 2);
+
+ XENBUS_STORE(Free,
+ &Frontend->StoreInterface,
+ Buffer);
+ } else {
+ Frontend->Split = FALSE;
+ }
+
+ Info("%s: %s\n", __FrontendGetPath(Frontend),
+ (Frontend->Split) ? "TRUE" : "FALSE");
+}
+
+static FORCEINLINE BOOLEAN
+__FrontendIsSplit(
+ IN PXENVIF_FRONTEND Frontend
+ )
+{
+ return Frontend->Split;
+}
+
+BOOLEAN
+FrontendIsSplit(
+ IN PXENVIF_FRONTEND Frontend
+ )
+{
+ return __FrontendIsSplit(Frontend);
+}
+
static FORCEINLINE NTSTATUS
__FrontendUpdateHash(
PXENVIF_FRONTEND Frontend,
goto fail3;
FrontendSetNumQueues(Frontend);
-
- status = PollerConnect(__FrontendGetPoller(Frontend));
- if (!NT_SUCCESS(status))
- goto fail4;
+ FrontendSetSplit(Frontend);
status = ReceiverConnect(__FrontendGetReceiver(Frontend));
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail4;
status = TransmitterConnect(__FrontendGetTransmitter(Frontend));
if (!NT_SUCCESS(status))
- goto fail6;
+ goto fail5;
status = ControllerConnect(__FrontendGetController(Frontend));
if (!NT_SUCCESS(status))
- goto fail7;
+ goto fail6;
Attempt = 0;
do {
if (!NT_SUCCESS(status))
break;
- status = PollerStoreWrite(__FrontendGetPoller(Frontend),
- Transaction);
- if (!NT_SUCCESS(status))
- goto abort;
-
status = ReceiverStoreWrite(__FrontendGetReceiver(Frontend),
Transaction);
if (!NT_SUCCESS(status))
} while (status == STATUS_RETRY);
if (!NT_SUCCESS(status))
- goto fail8;
+ goto fail7;
State = XenbusStateUnknown;
while (State != XenbusStateConnected) {
status = STATUS_UNSUCCESSFUL;
if (State != XenbusStateConnected)
- goto fail9;
+ goto fail8;
ControllerEnable(__FrontendGetController(Frontend));
Trace("<====\n");
return STATUS_SUCCESS;
-fail9:
- Error("fail9\n");
-
fail8:
Error("fail8\n");
- ControllerDisconnect(__FrontendGetController(Frontend));
-
fail7:
Error("fail7\n");
- TransmitterDisconnect(__FrontendGetTransmitter(Frontend));
+ ControllerDisconnect(__FrontendGetController(Frontend));
fail6:
Error("fail6\n");
- ReceiverDisconnect(__FrontendGetReceiver(Frontend));
+ TransmitterDisconnect(__FrontendGetTransmitter(Frontend));
fail5:
Error("fail5\n");
- PollerDisconnect(__FrontendGetPoller(Frontend));
+ ReceiverDisconnect(__FrontendGetReceiver(Frontend));
fail4:
Error("fail4\n");
MacDisconnect(__FrontendGetMac(Frontend));
+ Frontend->Split = FALSE;
Frontend->NumQueues = 0;
fail3:
ControllerDisconnect(__FrontendGetController(Frontend));
TransmitterDisconnect(__FrontendGetTransmitter(Frontend));
ReceiverDisconnect(__FrontendGetReceiver(Frontend));
- PollerDisconnect(__FrontendGetPoller(Frontend));
MacDisconnect(__FrontendGetMac(Frontend));
+ Frontend->Split = FALSE;
Frontend->NumQueues = 0;
XENBUS_DEBUG(Deregister,
if (!NT_SUCCESS(status))
goto fail1;
- status = PollerEnable(__FrontendGetPoller(Frontend));
- if (!NT_SUCCESS(status))
- goto fail2;
-
status = ReceiverEnable(__FrontendGetReceiver(Frontend));
if (!NT_SUCCESS(status))
- goto fail3;
+ goto fail2;
status = TransmitterEnable(__FrontendGetTransmitter(Frontend));
if (!NT_SUCCESS(status))
- goto fail4;
+ goto fail3;
status = __FrontendUpdateHash(Frontend, &Frontend->Hash);
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail4;
(VOID) FrontendNotifyMulticastAddresses(Frontend, TRUE);
Trace("<====\n");
return STATUS_SUCCESS;
-fail5:
- Error("fail5\n");
-
- TransmitterDisable(__FrontendGetTransmitter(Frontend));
-
fail4:
Error("fail4\n");
- ReceiverDisable(__FrontendGetReceiver(Frontend));
+ TransmitterDisable(__FrontendGetTransmitter(Frontend));
fail3:
Error("fail3\n");
- PollerDisable(__FrontendGetPoller(Frontend));
+ ReceiverDisable(__FrontendGetReceiver(Frontend));
fail2:
Error("fail2\n");
TransmitterDisable(__FrontendGetTransmitter(Frontend));
ReceiverDisable(__FrontendGetReceiver(Frontend));
- PollerDisable(__FrontendGetPoller(Frontend));
MacDisable(__FrontendGetMac(Frontend));
Trace("<====\n");
if (!NT_SUCCESS(status))
goto fail8;
- status = PollerInitialize(*Frontend, &(*Frontend)->Poller);
- if (!NT_SUCCESS(status))
- goto fail9;
-
status = ControllerInitialize(*Frontend, &(*Frontend)->Controller);
if (!NT_SUCCESS(status))
- goto fail10;
+ goto fail9;
KeInitializeEvent(&(*Frontend)->EjectEvent, NotificationEvent, FALSE);
status = ThreadCreate(FrontendEject, *Frontend, &(*Frontend)->EjectThread);
if (!NT_SUCCESS(status))
- goto fail11;
+ goto fail10;
status = ThreadCreate(FrontendMib, *Frontend, &(*Frontend)->MibThread);
if (!NT_SUCCESS(status))
- goto fail12;
+ goto fail11;
(*Frontend)->StatisticsCount = KeQueryMaximumProcessorCountEx(ALL_PROCESSOR_GROUPS);
(*Frontend)->Statistics = __FrontendAllocate(sizeof (XENVIF_FRONTEND_STATISTICS) *
status = STATUS_NO_MEMORY;
if ((*Frontend)->Statistics == NULL)
- goto fail13;
+ goto fail12;
Trace("<====\n");
return STATUS_SUCCESS;
-fail13:
- Error("fail13\n");
+fail12:
+ Error("fail12\n");
ThreadAlert((*Frontend)->MibThread);
ThreadJoin((*Frontend)->MibThread);
(*Frontend)->MibThread = NULL;
-fail12:
- Error("fail12\n");
+fail11:
+ Error("fail11\n");
ThreadAlert((*Frontend)->EjectThread);
ThreadJoin((*Frontend)->EjectThread);
(*Frontend)->EjectThread = NULL;
-fail11:
- Error("fail11\n");
+fail10:
+ Error("fail10\n");
RtlZeroMemory(&(*Frontend)->EjectEvent, sizeof (KEVENT));
ControllerTeardown(__FrontendGetController(*Frontend));
(*Frontend)->Controller = NULL;
-fail10:
- PollerTeardown(__FrontendGetPoller(*Frontend));
- (*Frontend)->Poller = NULL;
-
fail9:
TransmitterTeardown(__FrontendGetTransmitter(*Frontend));
(*Frontend)->Transmitter = NULL;
ControllerTeardown(__FrontendGetController(Frontend));
Frontend->Controller = NULL;
- PollerTeardown(__FrontendGetPoller(Frontend));
- Frontend->Poller = NULL;
-
TransmitterTeardown(__FrontendGetTransmitter(Frontend));
Frontend->Transmitter = NULL;
IN PXENVIF_FRONTEND Frontend
);
+extern BOOLEAN
+FrontendIsSplit(
+ IN PXENVIF_FRONTEND Frontend
+ );
+
+extern BOOLEAN
+FrontendIsSplit(
+ IN PXENVIF_FRONTEND Frontend
+ );
+
extern PCHAR
FrontendFormatPath(
IN PXENVIF_FRONTEND Frontend,
IN PXENVIF_FRONTEND Frontend
);
-#include "poller.h"
-
-extern PXENVIF_POLLER
-FrontendGetPoller(
- IN PXENVIF_FRONTEND Frontend
- );
-
#include "controller.h"
extern PXENVIF_CONTROLLER
+++ /dev/null
-/* Copyright (c) Citrix Systems Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms,
- * with or without modification, are permitted provided
- * that the following conditions are met:
- *
- * * Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the
- * following disclaimer.
- * * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the
- * following disclaimer in the documetation and/or other
- * materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <ntddk.h>
-#include <procgrp.h>
-#include <ntstrsafe.h>
-#include <stdarg.h>
-#include <stdlib.h>
-#include <xen.h>
-
-#include <debug_interface.h>
-#include <store_interface.h>
-#include <evtchn_interface.h>
-
-#include "pdo.h"
-#include "frontend.h"
-#include "transmitter.h"
-#include "receiver.h"
-#include "poller.h"
-#include "vif.h"
-#include "thread.h"
-#include "registry.h"
-#include "dbg_print.h"
-#include "assert.h"
-#include "util.h"
-
-#define MAXNAMELEN 128
-
-typedef struct _XENVIF_POLLER_INSTANCE XENVIF_POLLER_INSTANCE, *PXENVIF_POLLER_INSTANCE;
-
-typedef enum _XENVIF_POLLER_CHANNEL_TYPE {
- XENVIF_POLLER_CHANNEL_RECEIVER,
- XENVIF_POLLER_CHANNEL_TRANSMITTER,
- XENVIF_POLLER_CHANNEL_COMBINED,
- XENVIF_POLLER_CHANNEL_TYPE_COUNT
-} XENVIF_POLLER_CHANNEL_TYPE, *PXENVIF_POLLER_CHANNEL_TYPE;
-
-#define XENVIF_POLLER_CHANNEL_INVALID XENVIF_POLLER_CHANNEL_TYPE_COUNT
-
-typedef struct _XENVIF_POLLER_CHANNEL {
- PXENVIF_POLLER_INSTANCE Instance;
- XENVIF_POLLER_CHANNEL_TYPE Type;
- const CHAR *Node;
- PXENBUS_EVTCHN_CHANNEL Channel;
- ULONG Events;
-} XENVIF_POLLER_CHANNEL, *PXENVIF_POLLER_CHANNEL;
-
-struct _XENVIF_POLLER_INSTANCE {
- PXENVIF_POLLER Poller;
- ULONG Index;
- PCHAR Path;
- KSPIN_LOCK Lock;
- KDPC Dpc;
- ULONG Dpcs;
- PXENVIF_POLLER_CHANNEL Channel[XENVIF_POLLER_CHANNEL_TYPE_COUNT];
- BOOLEAN Enabled;
- LONG Pending;
-};
-
-struct _XENVIF_POLLER {
- PXENVIF_FRONTEND Frontend;
- PXENVIF_POLLER_INSTANCE *Instance;
- BOOLEAN Split;
- XENBUS_STORE_INTERFACE StoreInterface;
- XENBUS_EVTCHN_INTERFACE EvtchnInterface;
- XENBUS_DEBUG_INTERFACE DebugInterface;
- PXENBUS_DEBUG_CALLBACK DebugCallback;
-};
-
-#define XENVIF_POLLER_TAG 'LLOP'
-
-static FORCEINLINE PVOID
-__PollerAllocate(
- IN ULONG Length
- )
-{
- return __AllocatePoolWithTag(NonPagedPool, Length, XENVIF_POLLER_TAG);
-}
-
-static FORCEINLINE VOID
-__PollerFree(
- IN PVOID Buffer
- )
-{
- __FreePoolWithTag(Buffer, XENVIF_POLLER_TAG);
-}
-
-static NTSTATUS
-PollerChannelInitialize(
- IN PXENVIF_POLLER_INSTANCE Instance,
- IN ULONG Type,
- OUT PXENVIF_POLLER_CHANNEL *Channel
- )
-{
- NTSTATUS status;
-
- *Channel = __PollerAllocate(sizeof (XENVIF_POLLER_CHANNEL));
-
- status = STATUS_NO_MEMORY;
- if (*Channel == NULL)
- goto fail1;
-
- (*Channel)->Instance = Instance;
- (*Channel)->Type = Type;
-
- switch (Type) {
- case XENVIF_POLLER_CHANNEL_RECEIVER:
- (*Channel)->Node = "event-channel-rx";
- break;
-
- case XENVIF_POLLER_CHANNEL_TRANSMITTER:
- (*Channel)->Node = "event-channel-tx";
- break;
-
- case XENVIF_POLLER_CHANNEL_COMBINED:
- (*Channel)->Node = "event-channel";
- break;
-
- default:
- ASSERT(FALSE);
- break;
- }
-
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-static BOOLEAN
-PollerChannelSetPending(
- IN PXENVIF_POLLER_CHANNEL Channel
- )
-{
- PXENVIF_POLLER_INSTANCE Instance;
- ULONG Set;
-
- Instance = Channel->Instance;
-
- switch (Channel->Type)
- {
- case XENVIF_POLLER_CHANNEL_RECEIVER:
- Set = InterlockedBitTestAndSet(&Instance->Pending,
- XENVIF_POLLER_EVENT_RECEIVE);
- break;
-
- case XENVIF_POLLER_CHANNEL_TRANSMITTER:
- Set = InterlockedBitTestAndSet(&Instance->Pending,
- XENVIF_POLLER_EVENT_TRANSMIT);
- break;
-
- case XENVIF_POLLER_CHANNEL_COMBINED:
- Set = InterlockedBitTestAndSet(&Instance->Pending,
- XENVIF_POLLER_EVENT_RECEIVE);
- Set |= InterlockedBitTestAndSet(&Instance->Pending,
- XENVIF_POLLER_EVENT_TRANSMIT);
- break;
-
- default:
- ASSERT(FALSE);
- Set = 0;
- break;
- }
-
- return (Set != 0) ? FALSE : TRUE;
-}
-
-static FORCEINLINE BOOLEAN
-__BitTest(
- IN PLONG Mask,
- IN LONG Bit
- )
-{
- return (*Mask & (1L << Bit)) ? TRUE : FALSE;
-}
-
-static BOOLEAN
-PollerChannelTestPending(
- IN PXENVIF_POLLER_CHANNEL Channel
- )
-{
- PXENVIF_POLLER_INSTANCE Instance;
-
- Instance = Channel->Instance;
-
- switch (Channel->Type)
- {
- case XENVIF_POLLER_CHANNEL_RECEIVER:
- if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_RECEIVE))
- return TRUE;
-
- break;
-
- case XENVIF_POLLER_CHANNEL_TRANSMITTER:
- if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_TRANSMIT))
- return TRUE;
-
- break;
-
- case XENVIF_POLLER_CHANNEL_COMBINED:
- if (__BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_RECEIVE) ||
- __BitTest(&Instance->Pending, XENVIF_POLLER_EVENT_TRANSMIT))
- return TRUE;
-
- break;
-
- default:
- ASSERT(FALSE);
- break;
- }
-
- return FALSE;
-}
-
-KSERVICE_ROUTINE PollerChannelEvtchnCallback;
-
-BOOLEAN
-PollerChannelEvtchnCallback(
- IN PKINTERRUPT InterruptObject,
- IN PVOID Argument
- )
-{
- PXENVIF_POLLER_CHANNEL Channel = Argument;
- PXENVIF_POLLER_INSTANCE Instance;
-
- UNREFERENCED_PARAMETER(InterruptObject);
-
- ASSERT(Channel != NULL);
- Instance = Channel->Instance;
-
- Channel->Events++;
-
- if (PollerChannelSetPending(Channel) &&
- KeInsertQueueDpc(&Instance->Dpc, NULL, NULL))
- Instance->Dpcs++;
-
- return TRUE;
-}
-
-static FORCEINLINE BOOLEAN
-__PollerIsSplit(
- IN PXENVIF_POLLER Poller
- )
-{
- return Poller->Split;
-}
-
-static NTSTATUS
-PollerChannelConnect(
- IN PXENVIF_POLLER_CHANNEL Channel
- )
-{
- PXENVIF_POLLER_INSTANCE Instance;
- PXENVIF_POLLER Poller;
- PXENVIF_FRONTEND Frontend;
- PROCESSOR_NUMBER ProcNumber;
- NTSTATUS status;
-
- Instance = Channel->Instance;
- Poller = Instance->Poller;
- Frontend = Poller->Frontend;
-
- switch (Channel->Type)
- {
- case XENVIF_POLLER_CHANNEL_RECEIVER:
- case XENVIF_POLLER_CHANNEL_TRANSMITTER:
- if (!__PollerIsSplit(Poller))
- goto done;
-
- break;
-
- case XENVIF_POLLER_CHANNEL_COMBINED:
- if (__PollerIsSplit(Poller))
- goto done;
-
- break;
-
- default:
- ASSERT(FALSE);
- break;
- }
-
- Channel->Channel = XENBUS_EVTCHN(Open,
- &Poller->EvtchnInterface,
- XENBUS_EVTCHN_TYPE_UNBOUND,
- PollerChannelEvtchnCallback,
- Channel,
- FrontendGetBackendDomain(Frontend),
- TRUE);
-
- status = STATUS_UNSUCCESSFUL;
- if (Channel->Channel == NULL)
- goto fail1;
-
- status = KeGetProcessorNumberFromIndex(Instance->Index, &ProcNumber);
- ASSERT(NT_SUCCESS(status));
-
- (VOID) XENBUS_EVTCHN(Bind,
- &Poller->EvtchnInterface,
- Channel->Channel,
- ProcNumber.Group,
- ProcNumber.Number);
-
- (VOID) XENBUS_EVTCHN(Unmask,
- &Poller->EvtchnInterface,
- Channel->Channel,
- FALSE,
- TRUE);
-
-done:
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-static NTSTATUS
-PollerChannelStoreWrite(
- IN PXENVIF_POLLER_CHANNEL Channel,
- IN PXENBUS_STORE_TRANSACTION Transaction
- )
-{
- PXENVIF_POLLER_INSTANCE Instance;
- PXENVIF_POLLER Poller;
- PCHAR Path;
- ULONG Port;
- NTSTATUS status;
-
- Instance = Channel->Instance;
- Poller = Instance->Poller;
-
- if (Channel->Channel == NULL)
- goto done;
-
- Path = (FrontendGetNumQueues(Poller->Frontend) == 1) ?
- FrontendGetPath(Poller->Frontend) :
- Instance->Path;
-
- Port = XENBUS_EVTCHN(GetPort,
- &Poller->EvtchnInterface,
- Channel->Channel);
-
- status = XENBUS_STORE(Printf,
- &Poller->StoreInterface,
- Transaction,
- Path,
- (PCHAR)Channel->Node,
- "%u",
- Port);
- if (!NT_SUCCESS(status))
- goto fail1;
-
-done:
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-static VOID
-PollerChannelUnmask(
- IN PXENVIF_POLLER_CHANNEL Channel
- )
-{
- PXENVIF_POLLER_INSTANCE Instance;
- PXENVIF_POLLER Poller;
- BOOLEAN Pending;
-
- Instance = Channel->Instance;
- Poller = Instance->Poller;
-
- if (Channel->Channel == NULL)
- return;
-
- if (PollerChannelTestPending(Channel))
- return;
-
- Pending = XENBUS_EVTCHN(Unmask,
- &Poller->EvtchnInterface,
- Channel->Channel,
- FALSE,
- FALSE);
- if (Pending)
- (VOID) PollerChannelSetPending(Channel);
-}
-
-static VOID
-PollerChannelSend(
- IN PXENVIF_POLLER_CHANNEL Channel
- )
-{
- PXENVIF_POLLER_INSTANCE Instance;
- PXENVIF_POLLER Poller;
-
- Instance = Channel->Instance;
- Poller = Instance->Poller;
-
- XENBUS_EVTCHN(Send,
- &Poller->EvtchnInterface,
- Channel->Channel);
-}
-
-static VOID
-PollerChannelDebugCallback(
- IN PXENVIF_POLLER_CHANNEL Channel
- )
-{
- PXENVIF_POLLER_INSTANCE Instance;
- PXENVIF_POLLER Poller;
-
- Instance = Channel->Instance;
- Poller = Instance->Poller;
-
- if (Channel->Channel == NULL)
- return;
-
- XENBUS_DEBUG(Printf,
- &Poller->DebugInterface,
- "[%s]: Events = %lu\n",
- Channel->Node,
- Channel->Events);
-}
-
-static VOID
-PollerChannelDisconnect(
- IN PXENVIF_POLLER_CHANNEL Channel
- )
-{
- PXENVIF_POLLER_INSTANCE Instance;
- PXENVIF_POLLER Poller;
-
- Instance = Channel->Instance;
- Poller = Instance->Poller;
-
- if (Channel->Channel == NULL)
- return;
-
- Channel->Events = 0;
-
- XENBUS_EVTCHN(Close,
- &Poller->EvtchnInterface,
- Channel->Channel);
- Channel->Channel = NULL;
-}
-
-static VOID
-PollerChannelTeardown(
- IN PXENVIF_POLLER_CHANNEL Channel
- )
-{
- Channel->Node = NULL;
-
- Channel->Type = 0;
- Channel->Instance = NULL;
-
- ASSERT(IsZeroMemory(Channel, sizeof (XENVIF_POLLER_CHANNEL)));
- __PollerFree(Channel);
-}
-
-__drv_requiresIRQL(DISPATCH_LEVEL)
-static VOID
-PollerInstanceUnmask(
- IN PXENVIF_POLLER_INSTANCE Instance,
- IN XENVIF_POLLER_EVENT_TYPE Event
- )
-{
- PXENVIF_POLLER Poller;
- XENVIF_POLLER_CHANNEL_TYPE Type;
- PXENVIF_POLLER_CHANNEL Channel;
-
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- Poller = Instance->Poller;
-
- KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
-
- if (!Instance->Enabled)
- goto done;
-
- if (!__PollerIsSplit(Poller)) {
- Type = XENVIF_POLLER_CHANNEL_COMBINED;
- } else {
- switch (Event) {
- case XENVIF_POLLER_EVENT_RECEIVE:
- Type = XENVIF_POLLER_CHANNEL_RECEIVER;
- break;
-
- case XENVIF_POLLER_EVENT_TRANSMIT:
- Type = XENVIF_POLLER_CHANNEL_TRANSMITTER;
- break;
-
- default:
- Type = XENVIF_POLLER_CHANNEL_INVALID;
- break;
- }
- }
-
- ASSERT(Type != XENVIF_POLLER_CHANNEL_INVALID);
-
- Channel = Instance->Channel[Type];
-
- PollerChannelUnmask(Channel);
-
-done:
- KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
-}
-
-__drv_functionClass(KDEFERRED_ROUTINE)
-__drv_maxIRQL(DISPATCH_LEVEL)
-__drv_minIRQL(DISPATCH_LEVEL)
-__drv_sameIRQL
-static VOID
-PollerInstanceDpc(
- IN PKDPC Dpc,
- IN PVOID Context,
- IN PVOID Argument1,
- IN PVOID Argument2
- )
-{
- PXENVIF_POLLER_INSTANCE Instance = Context;
- PXENVIF_POLLER Poller;
- PXENVIF_FRONTEND Frontend;
- BOOLEAN Enabled;
- BOOLEAN ReceiverRetry;
- BOOLEAN TransmitterRetry;
-
- UNREFERENCED_PARAMETER(Dpc);
- UNREFERENCED_PARAMETER(Argument1);
- UNREFERENCED_PARAMETER(Argument2);
-
- ASSERT(Instance != NULL);
-
- Poller = Instance->Poller;
- Frontend = Poller->Frontend;
- Enabled = FALSE;
- ReceiverRetry = FALSE;
- TransmitterRetry = FALSE;
-
- for (;;) {
- BOOLEAN NeedReceiverPoll;
- BOOLEAN NeedTransmitterPoll;
-
- KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
- Enabled = Instance->Enabled;
- KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
-
- if (!Enabled)
- break;
-
- NeedReceiverPoll =
- (InterlockedBitTestAndReset(&Instance->Pending,
- XENVIF_POLLER_EVENT_RECEIVE) != 0) ?
- TRUE :
- FALSE;
-
- NeedTransmitterPoll =
- (InterlockedBitTestAndReset(&Instance->Pending,
- XENVIF_POLLER_EVENT_TRANSMIT) != 0) ?
- TRUE :
- FALSE;
-
- if (!NeedReceiverPoll && !NeedTransmitterPoll)
- break;
-
- if (NeedReceiverPoll)
- {
- ReceiverRetry = ReceiverPoll(FrontendGetReceiver(Frontend),
- Instance->Index);
-
- if (!ReceiverRetry) {
- PollerInstanceUnmask(Instance, XENVIF_POLLER_EVENT_RECEIVE);
- } else {
- (VOID) InterlockedBitTestAndSet(&Instance->Pending,
- XENVIF_POLLER_EVENT_RECEIVE);
- }
- }
-
- if (NeedTransmitterPoll)
- {
- TransmitterRetry = TransmitterPoll(FrontendGetTransmitter(Frontend),
- Instance->Index);
-
- if (!TransmitterRetry) {
- PollerInstanceUnmask(Instance, XENVIF_POLLER_EVENT_TRANSMIT);
- } else {
- (VOID) InterlockedBitTestAndSet(&Instance->Pending,
- XENVIF_POLLER_EVENT_TRANSMIT);
- }
- }
- }
-
- ASSERT(!Enabled || !ReceiverRetry);
- ASSERT(!Enabled || !TransmitterRetry);
-}
-
-static NTSTATUS
-PollerInstanceInitialize(
- IN PXENVIF_POLLER Poller,
- IN LONG Index,
- OUT PXENVIF_POLLER_INSTANCE *Instance
- )
-{
- PXENVIF_FRONTEND Frontend;
- LONG Type;
- NTSTATUS status;
-
- Frontend = Poller->Frontend;
-
- *Instance = __PollerAllocate(sizeof (XENVIF_POLLER_INSTANCE));
-
- status = STATUS_NO_MEMORY;
- if (*Instance == NULL)
- goto fail1;
-
- (*Instance)->Poller = Poller;
- (*Instance)->Index = Index;
-
- for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
- {
- PXENVIF_POLLER_CHANNEL Channel;
-
- status = PollerChannelInitialize(*Instance, Type, &Channel);
- if (!NT_SUCCESS(status))
- goto fail2;
-
- (*Instance)->Channel[Type] = Channel;
- }
-
- (*Instance)->Path = FrontendFormatPath(Frontend, Index);
- if ((*Instance)->Path == NULL)
- goto fail3;
-
- KeInitializeSpinLock(&(*Instance)->Lock);
-
- KeInitializeDpc(&(*Instance)->Dpc, PollerInstanceDpc, *Instance);
-
- return STATUS_SUCCESS;
-
-fail3:
- Error("fail3\n");
-
- Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
-
-fail2:
- Error("fail2\n");
-
- while (--Type >= 0)
- {
- PXENVIF_POLLER_CHANNEL Channel = (*Instance)->Channel[Type];
-
- (*Instance)->Channel[Type] = NULL;
- PollerChannelTeardown(Channel);
- }
-
- (*Instance)->Index = 0;
- (*Instance)->Poller = NULL;
-
- ASSERT(IsZeroMemory(*Instance, sizeof (XENVIF_POLLER_INSTANCE)));
- __PollerFree(*Instance);
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-__drv_requiresIRQL(DISPATCH_LEVEL)
-static NTSTATUS
-PollerInstanceConnect(
- IN PXENVIF_POLLER_INSTANCE Instance
- )
-{
- PROCESSOR_NUMBER ProcNumber;
- LONG Type;
- NTSTATUS status;
-
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- status = KeGetProcessorNumberFromIndex(Instance->Index, &ProcNumber);
- ASSERT(NT_SUCCESS(status));
-
- KeSetTargetProcessorDpcEx(&Instance->Dpc, &ProcNumber);
- KeSetImportanceDpc(&Instance->Dpc, MediumHighImportance);
-
- for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
- {
- PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
-
- status = PollerChannelConnect(Channel);
- if (!NT_SUCCESS(status))
- goto fail1;
- }
-
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- while (--Type >= 0)
- {
- PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
-
- PollerChannelDisconnect(Channel);
- }
-
- return status;
-}
-
-static NTSTATUS
-PollerInstanceStoreWrite(
- IN PXENVIF_POLLER_INSTANCE Instance,
- IN PXENBUS_STORE_TRANSACTION Transaction
- )
-{
- ULONG Type;
- NTSTATUS status;
-
- for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
- {
- PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
-
- status = PollerChannelStoreWrite(Channel, Transaction);
- if (!NT_SUCCESS(status))
- goto fail1;
- }
-
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-__drv_requiresIRQL(DISPATCH_LEVEL)
-static NTSTATUS
-PollerInstanceEnable(
- IN PXENVIF_POLLER_INSTANCE Instance
- )
-{
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- (VOID) InterlockedBitTestAndSet(&Instance->Pending,
- XENVIF_POLLER_EVENT_RECEIVE);
- (VOID) InterlockedBitTestAndSet(&Instance->Pending,
- XENVIF_POLLER_EVENT_TRANSMIT);
-
- KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
- Instance->Enabled = TRUE;
- KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
-
- (VOID) KeInsertQueueDpc(&Instance->Dpc, NULL, NULL);
-
- return STATUS_SUCCESS;
-}
-
-__drv_requiresIRQL(DISPATCH_LEVEL)
-static NTSTATUS
-PollerInstanceSend(
- IN PXENVIF_POLLER_INSTANCE Instance,
- IN XENVIF_POLLER_EVENT_TYPE Event
- )
-{
- PXENVIF_POLLER Poller;
- XENVIF_POLLER_CHANNEL_TYPE Type;
- PXENVIF_POLLER_CHANNEL Channel;
- NTSTATUS status;
-
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- Poller = Instance->Poller;
-
- KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
-
- Type = XENVIF_POLLER_CHANNEL_INVALID;
-
- if (Instance->Enabled) {
- if (!__PollerIsSplit(Poller)) {
- Type = XENVIF_POLLER_CHANNEL_COMBINED;
- } else {
- switch (Event) {
- case XENVIF_POLLER_EVENT_RECEIVE:
- Type = XENVIF_POLLER_CHANNEL_RECEIVER;
- break;
-
- case XENVIF_POLLER_EVENT_TRANSMIT:
- Type = XENVIF_POLLER_CHANNEL_TRANSMITTER;
- break;
-
- default:
- ASSERT(FALSE);
- break;
- }
- }
- }
-
- KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
-
- status = STATUS_UNSUCCESSFUL;
- if (Type == XENVIF_POLLER_CHANNEL_INVALID)
- goto fail1;
-
- Channel = Instance->Channel[Type];
-
- PollerChannelSend(Channel);
-
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-__drv_requiresIRQL(DISPATCH_LEVEL)
-static NTSTATUS
-PollerInstanceTrigger(
- IN PXENVIF_POLLER_INSTANCE Instance,
- IN XENVIF_POLLER_EVENT_TYPE Event
- )
-{
- NTSTATUS status;
-
- status = STATUS_INVALID_PARAMETER;
- if (Event >= XENVIF_POLLER_EVENT_TYPE_COUNT)
- goto fail1;
-
- (VOID) InterlockedBitTestAndSet(&Instance->Pending, Event);
-
- if (KeInsertQueueDpc(&Instance->Dpc, NULL, NULL))
- Instance->Dpcs++;
-
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-static VOID
-PollerInstanceDebugCallback(
- IN PXENVIF_POLLER_INSTANCE Instance
- )
-{
- PXENVIF_POLLER Poller;
- ULONG Type;
-
- Poller = Instance->Poller;
-
- XENBUS_DEBUG(Printf,
- &Poller->DebugInterface,
- "[%d]: Dpcs = %lu\n",
- Instance->Index,
- Instance->Dpcs);
-
- for (Type = 0; Type < XENVIF_POLLER_CHANNEL_TYPE_COUNT; Type++)
- {
- PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
-
- PollerChannelDebugCallback(Channel);
- }
-}
-
-__drv_requiresIRQL(DISPATCH_LEVEL)
-static VOID
-PollerInstanceDisable(
- IN PXENVIF_POLLER_INSTANCE Instance
- )
-{
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- KeAcquireSpinLockAtDpcLevel(&Instance->Lock);
- Instance->Enabled = FALSE;
- KeReleaseSpinLockFromDpcLevel(&Instance->Lock);
-}
-
-__drv_requiresIRQL(DISPATCH_LEVEL)
-static VOID
-PollerInstanceDisconnect(
- IN PXENVIF_POLLER_INSTANCE Instance
- )
-{
- LONG Type;
-
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- Instance->Dpcs = 0;
- Instance->Pending = 0;
-
- Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
-
- while (--Type >= 0)
- {
- PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
-
- PollerChannelDisconnect(Channel);
- }
-}
-
-static VOID
-PollerInstanceTeardown(
- IN PXENVIF_POLLER_INSTANCE Instance
- )
-{
- PXENVIF_POLLER Poller;
- PXENVIF_FRONTEND Frontend;
- LONG Type;
-
- ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
- KeFlushQueuedDpcs();
-
- Poller = Instance->Poller;
- Frontend = Poller->Frontend;
-
- RtlZeroMemory(&Instance->Dpc, sizeof (KDPC));
-
- RtlZeroMemory(&Instance->Lock, sizeof (KSPIN_LOCK));
-
- FrontendFreePath(Frontend, Instance->Path);
- Instance->Path = NULL;
-
- Type = XENVIF_POLLER_CHANNEL_TYPE_COUNT;
-
- while (--Type >= 0)
- {
- PXENVIF_POLLER_CHANNEL Channel = Instance->Channel[Type];
-
- Instance->Channel[Type] = NULL;
- PollerChannelTeardown(Channel);
- }
-
- Instance->Index = 0;
- Instance->Poller = NULL;
-
- ASSERT(IsZeroMemory(Instance, sizeof (XENVIF_POLLER_INSTANCE)));
- __PollerFree(Instance);
-}
-
-static VOID
-PollerDebugCallback(
- IN PVOID Argument,
- IN BOOLEAN Crashing
- )
-{
- PXENVIF_POLLER Poller = Argument;
- PXENVIF_FRONTEND Frontend;
- ULONG NumQueues;
- ULONG Index;
-
- UNREFERENCED_PARAMETER(Crashing);
-
- Frontend = Poller->Frontend;
-
- NumQueues = FrontendGetNumQueues(Frontend);
-
- for (Index = 0; Index < NumQueues; Index++) {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- PollerInstanceDebugCallback(Instance);
- }
-}
-
-static VOID
-PollerSetSplit(
- IN PXENVIF_POLLER Poller
- )
-{
- PXENVIF_FRONTEND Frontend;
- PCHAR Buffer;
- NTSTATUS status;
-
- Frontend = Poller->Frontend;
-
- status = XENBUS_STORE(Read,
- &Poller->StoreInterface,
- NULL,
- FrontendGetBackendPath(Frontend),
- "feature-split-event-channels",
- &Buffer);
- if (NT_SUCCESS(status)) {
- Poller->Split = (BOOLEAN)strtol(Buffer, NULL, 2);
-
- XENBUS_STORE(Free,
- &Poller->StoreInterface,
- Buffer);
- } else {
- Poller->Split = FALSE;
- }
-
- Info("%s: %s\n", FrontendGetPath(Frontend),
- (Poller->Split) ? "TRUE" : "FALSE");
-}
-
-NTSTATUS
-PollerInitialize(
- IN PXENVIF_FRONTEND Frontend,
- OUT PXENVIF_POLLER *Poller
- )
-{
- LONG MaxQueues;
- LONG Index;
- NTSTATUS status;
-
- *Poller = __PollerAllocate(sizeof (XENVIF_POLLER));
-
- status = STATUS_NO_MEMORY;
- if (*Poller == NULL)
- goto fail1;
-
- FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
- &(*Poller)->EvtchnInterface);
-
- FdoGetStoreInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
- &(*Poller)->StoreInterface);
-
- FdoGetDebugInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
- &(*Poller)->DebugInterface);
-
- (*Poller)->Frontend = Frontend;
-
- MaxQueues = FrontendGetMaxQueues(Frontend);
- (*Poller)->Instance = __PollerAllocate(sizeof (PXENVIF_POLLER_INSTANCE) *
- MaxQueues);
-
- status = STATUS_NO_MEMORY;
- if ((*Poller)->Instance == NULL)
- goto fail2;
-
- for (Index = 0; Index < MaxQueues; Index++) {
- PXENVIF_POLLER_INSTANCE Instance;
-
- status = PollerInstanceInitialize(*Poller, Index, &Instance);
- if (!NT_SUCCESS(status))
- goto fail3;
-
- (*Poller)->Instance[Index] = Instance;
- }
-
- return STATUS_SUCCESS;
-
-fail3:
- Error("fail3\n");
-
- while (--Index >= 0)
- {
- PXENVIF_POLLER_INSTANCE Instance = (*Poller)->Instance[Index];
-
- (*Poller)->Instance[Index] = NULL;
- PollerInstanceTeardown(Instance);
- }
-
- ASSERT(IsZeroMemory((*Poller)->Instance,
- sizeof (PXENVIF_POLLER_INSTANCE) * MaxQueues));
- __PollerFree((*Poller)->Instance);
- (*Poller)->Instance = NULL;
-
-fail2:
- Error("fail2\n");
-
- (*Poller)->Frontend = NULL;
-
- RtlZeroMemory(&(*Poller)->DebugInterface,
- sizeof (XENBUS_DEBUG_INTERFACE));
-
- RtlZeroMemory(&(*Poller)->StoreInterface,
- sizeof (XENBUS_STORE_INTERFACE));
-
- RtlZeroMemory(&(*Poller)->EvtchnInterface,
- sizeof (XENBUS_EVTCHN_INTERFACE));
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-NTSTATUS
-PollerConnect(
- IN PXENVIF_POLLER Poller
- )
-{
- PXENVIF_FRONTEND Frontend;
- LONG NumQueues;
- LONG Index;
- NTSTATUS status;
-
- Trace("====>\n");
-
- Frontend = Poller->Frontend;
-
- status = XENBUS_EVTCHN(Acquire, &Poller->EvtchnInterface);
- if (!NT_SUCCESS(status))
- goto fail1;
-
- status = XENBUS_STORE(Acquire, &Poller->StoreInterface);
- if (!NT_SUCCESS(status))
- goto fail2;
-
- status = XENBUS_DEBUG(Acquire, &Poller->DebugInterface);
- if (!NT_SUCCESS(status))
- goto fail3;
-
- PollerSetSplit(Poller);
-
- NumQueues = FrontendGetNumQueues(Frontend);
-
- for (Index = 0; Index < NumQueues; Index++) {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- status = PollerInstanceConnect(Instance);
- if (!NT_SUCCESS(status))
- goto fail4;
- }
-
- status = XENBUS_DEBUG(Register,
- &Poller->DebugInterface,
- __MODULE__ "|POLLER",
- PollerDebugCallback,
- Poller,
- &Poller->DebugCallback);
- if (!NT_SUCCESS(status))
- goto fail5;
-
- Trace("<====\n");
- return STATUS_SUCCESS;
-
-fail5:
- Error("fail5\n");
-
- Index = NumQueues;
-
-fail4:
- Error("fail4\n");
-
- while (--Index >= 0)
- {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- PollerInstanceDisconnect(Instance);
- }
-
- Poller->Split = FALSE;
-
- XENBUS_DEBUG(Release, &Poller->DebugInterface);
-
-fail3:
- Error("fail3\n");
-
- XENBUS_STORE(Release, &Poller->StoreInterface);
-
-fail2:
- Error("fail2\n");
-
- XENBUS_EVTCHN(Release, &Poller->EvtchnInterface);
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-NTSTATUS
-PollerStoreWrite(
- IN PXENVIF_POLLER Poller,
- IN PXENBUS_STORE_TRANSACTION Transaction
- )
-{
- PXENVIF_FRONTEND Frontend;
- LONG NumQueues;
- LONG Index;
- NTSTATUS status;
-
- Trace("====>\n");
-
- Frontend = Poller->Frontend;
-
- NumQueues = FrontendGetNumQueues(Frontend);
-
- for (Index = 0; Index < NumQueues; Index++) {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- status = PollerInstanceStoreWrite(Instance, Transaction);
- if (!NT_SUCCESS(status))
- goto fail1;
- }
-
- Trace("<====\n");
-
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-NTSTATUS
-PollerEnable(
- IN PXENVIF_POLLER Poller
- )
-{
- PXENVIF_FRONTEND Frontend;
- LONG NumQueues;
- LONG Index;
- NTSTATUS status;
-
- Trace("====>\n");
-
- Frontend = Poller->Frontend;
-
- NumQueues = FrontendGetNumQueues(Frontend);
-
- for (Index = 0; Index < NumQueues; Index++) {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- status = PollerInstanceEnable(Instance);
- if (!NT_SUCCESS(status))
- goto fail1;
- }
-
- Trace("<====\n");
-
- return STATUS_SUCCESS;
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- while (--Index >= 0)
- {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- PollerInstanceDisable(Instance);
- }
-
- return status;
-}
-
-NTSTATUS
-PollerSend(
- IN PXENVIF_POLLER Poller,
- IN ULONG Index,
- IN XENVIF_POLLER_EVENT_TYPE Event
- )
-{
- PXENVIF_FRONTEND Frontend;
- ULONG NumQueues;
- PXENVIF_POLLER_INSTANCE Instance;
- NTSTATUS status;
-
- Frontend = Poller->Frontend;
-
- NumQueues = FrontendGetNumQueues(Frontend);
-
- status = STATUS_INVALID_PARAMETER;
- if (Index >= NumQueues)
- goto fail1;
-
- Instance = Poller->Instance[Index];
-
- status = PollerInstanceSend(Instance, Event);
- if (!NT_SUCCESS(status))
- goto fail2;
-
- return STATUS_SUCCESS;
-
-fail2:
- Error("fail2\n");
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-NTSTATUS
-PollerTrigger(
- IN PXENVIF_POLLER Poller,
- IN ULONG Index,
- IN XENVIF_POLLER_EVENT_TYPE Event
- )
-{
- PXENVIF_FRONTEND Frontend;
- ULONG NumQueues;
- PXENVIF_POLLER_INSTANCE Instance;
- NTSTATUS status;
-
- Frontend = Poller->Frontend;
-
- NumQueues = FrontendGetNumQueues(Frontend);
-
- status = STATUS_INVALID_PARAMETER;
- if (Index >= NumQueues)
- goto fail1;
-
- Instance = Poller->Instance[Index];
-
- status = PollerInstanceTrigger(Instance, Event);
- if (!NT_SUCCESS(status))
- goto fail2;
-
- return STATUS_SUCCESS;
-
-fail2:
- Error("fail2\n");
-
-fail1:
- Error("fail1 (%08x)\n", status);
-
- return status;
-}
-
-VOID
-PollerDisable(
- IN PXENVIF_POLLER Poller
- )
-{
- PXENVIF_FRONTEND Frontend;
- LONG NumQueues;
- LONG Index;
-
- Trace("====>\n");
-
- Frontend = Poller->Frontend;
-
- NumQueues = FrontendGetNumQueues(Frontend);
- Index = NumQueues;
-
- while (--Index >= 0)
- {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- PollerInstanceDisable(Instance);
- }
-
- Trace("<====\n");
-}
-
-VOID
-PollerDisconnect(
- IN PXENVIF_POLLER Poller
- )
-{
- PXENVIF_FRONTEND Frontend;
- LONG NumQueues;
- LONG Index;
-
- Trace("====>\n");
-
- Frontend = Poller->Frontend;
-
- XENBUS_DEBUG(Deregister,
- &Poller->DebugInterface,
- Poller->DebugCallback);
- Poller->DebugCallback = NULL;
-
- NumQueues = FrontendGetNumQueues(Frontend);
- Index = NumQueues;
-
- while (--Index >= 0)
- {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- PollerInstanceDisconnect(Instance);
- }
-
- Poller->Split = FALSE;
-
- XENBUS_DEBUG(Release, &Poller->DebugInterface);
-
- XENBUS_STORE(Release, &Poller->StoreInterface);
-
- XENBUS_EVTCHN(Release, &Poller->EvtchnInterface);
-
- Trace("<====\n");
-}
-
-VOID
-PollerTeardown(
- IN PXENVIF_POLLER Poller
- )
-{
- PXENVIF_FRONTEND Frontend;
- LONG MaxQueues;
- LONG Index;
-
- ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
-
- Frontend = Poller->Frontend;
-
- MaxQueues = FrontendGetMaxQueues(Frontend);
- Index = MaxQueues;
-
- while (--Index >= 0)
- {
- PXENVIF_POLLER_INSTANCE Instance = Poller->Instance[Index];
-
- Poller->Instance[Index] = NULL;
- PollerInstanceTeardown(Instance);
- }
-
- ASSERT(IsZeroMemory(Poller->Instance,
- sizeof (PXENVIF_POLLER_INSTANCE) * MaxQueues));
- __PollerFree(Poller->Instance);
- Poller->Instance = NULL;
-
- Poller->Frontend = NULL;
-
- RtlZeroMemory(&Poller->DebugInterface,
- sizeof (XENBUS_DEBUG_INTERFACE));
-
- RtlZeroMemory(&Poller->StoreInterface,
- sizeof (XENBUS_STORE_INTERFACE));
-
- RtlZeroMemory(&Poller->EvtchnInterface,
- sizeof (XENBUS_EVTCHN_INTERFACE));
-
- ASSERT(IsZeroMemory(Poller, sizeof (XENVIF_POLLER)));
- __PollerFree(Poller);
-}
+++ /dev/null
-/* Copyright (c) Citrix Systems Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms,
- * with or without modification, are permitted provided
- * that the following conditions are met:
- *
- * * Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the
- * following disclaimer.
- * * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the
- * following disclaimer in the documentation and/or other
- * materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
- * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifndef _XENVIF_POLLER_H
-#define _XENVIF_POLLER_H
-
-#include <ntddk.h>
-
-#include <vif_interface.h>
-
-#include "frontend.h"
-
-typedef struct _XENVIF_POLLER XENVIF_POLLER, *PXENVIF_POLLER;
-
-typedef enum _XENVIF_POLLER_EVENT_TYPE {
- XENVIF_POLLER_EVENT_RECEIVE,
- XENVIF_POLLER_EVENT_TRANSMIT,
- XENVIF_POLLER_EVENT_TYPE_COUNT
-} XENVIF_POLLER_EVENT_TYPE, *PXENVIF_POLLER_EVENT_TYPE;
-
-extern NTSTATUS
-PollerInitialize(
- IN PXENVIF_FRONTEND Frontend,
- OUT PXENVIF_POLLER *Poller
- );
-
-extern NTSTATUS
-PollerConnect(
- IN PXENVIF_POLLER Poller
- );
-
-extern NTSTATUS
-PollerStoreWrite(
- IN PXENVIF_POLLER Poller,
- IN PXENBUS_STORE_TRANSACTION Transaction
- );
-
-extern NTSTATUS
-PollerEnable(
- IN PXENVIF_POLLER Poller
- );
-
-extern NTSTATUS
-PollerSend(
- IN PXENVIF_POLLER Poller,
- IN ULONG Index,
- IN XENVIF_POLLER_EVENT_TYPE Event
- );
-
-extern NTSTATUS
-PollerTrigger(
- IN PXENVIF_POLLER Poller,
- IN ULONG Index,
- IN XENVIF_POLLER_EVENT_TYPE Event
- );
-
-extern VOID
-PollerDisable(
- IN PXENVIF_POLLER Poller
- );
-
-extern VOID
-PollerDisconnect(
- IN PXENVIF_POLLER Poller
- );
-
-extern VOID
-PollerTeardown(
- IN PXENVIF_POLLER Poller
- );
-
-#endif // _XENVIF_POLLER_H
#include <store_interface.h>
#include <cache_interface.h>
#include <gnttab_interface.h>
+#include <evtchn_interface.h>
#include "pdo.h"
#include "registry.h"
netif_rx_front_ring_t Front;
netif_rx_sring_t *Shared;
PXENBUS_GNTTAB_ENTRY Entry;
+ PXENBUS_EVTCHN_CHANNEL Channel;
+ KDPC Dpc;
+ ULONG Dpcs;
+ KTIMER Timer;
+ KDPC TimerDpc;
+ ULONG Events;
PXENVIF_RECEIVER_FRAGMENT Pending[XENVIF_RECEIVER_MAXIMUM_FRAGMENT_ID + 1];
ULONG RequestsPosted;
ULONG RequestsPushed;
ULONG BackfillSize;
PXENBUS_DEBUG_CALLBACK DebugCallback;
PXENVIF_THREAD WatchdogThread;
- PLIST_ENTRY PacketQueue;
- KDPC Dpc;
- ULONG Dpcs;
- LIST_ENTRY PacketComplete;
+ LIST_ENTRY PacketList;
XENVIF_RECEIVER_HASH Hash;
} XENVIF_RECEIVER_RING, *PXENVIF_RECEIVER_RING;
PXENVIF_FRONTEND Frontend;
XENBUS_CACHE_INTERFACE CacheInterface;
XENBUS_GNTTAB_INTERFACE GnttabInterface;
+ XENBUS_EVTCHN_INTERFACE EvtchnInterface;
PXENVIF_RECEIVER_RING *Ring;
LONG Loaned;
LONG Returned;
return NULL;
}
-static VOID
-ReceiverRingCompletePacket(
- IN PXENVIF_RECEIVER_RING Ring,
- IN PXENVIF_RECEIVER_PACKET Packet
- )
-{
- ReceiverRingProcessTag(Ring, Packet);
- ReceiverRingProcessChecksum(Ring, Packet);
-
- ASSERT(IsZeroMemory(&Packet->ListEntry, sizeof (LIST_ENTRY)));
- InsertTailList(&Ring->PacketComplete, &Packet->ListEntry);
-}
-
static VOID
ReceiverRingProcessLargePacket(
IN PXENVIF_RECEIVER_RING Ring,
- IN PXENVIF_RECEIVER_PACKET Packet
+ IN PXENVIF_RECEIVER_PACKET Packet,
+ OUT PLIST_ENTRY List
)
{
PXENVIF_RECEIVER Receiver;
ASSERT3U(Length, >=, SegmentSize);
Length -= SegmentSize;
- ReceiverRingCompletePacket(Ring, Segment);
+ ASSERT(IsZeroMemory(&Segment->ListEntry, sizeof (LIST_ENTRY)));
+ InsertTailList(List, &Segment->ListEntry);
if (Offload) {
ASSERT(Ring->OffloadOptions.NeedLargePacketSplit != 0);
if (Receiver->AlwaysPullup != 0)
__ReceiverRingPullupPacket(Ring, Packet);
- ReceiverRingCompletePacket(Ring, Packet);
+ ASSERT(IsZeroMemory(&Packet->ListEntry, sizeof (LIST_ENTRY)));
+ InsertTailList(List, &Packet->ListEntry);
} else {
__ReceiverRingPutPacket(Ring, Packet, TRUE);
}
static VOID
ReceiverRingProcessStandardPacket(
IN PXENVIF_RECEIVER_RING Ring,
- IN PXENVIF_RECEIVER_PACKET Packet
+ IN PXENVIF_RECEIVER_PACKET Packet,
+ OUT PLIST_ENTRY List
)
{
PXENVIF_RECEIVER Receiver;
Packet->Mdl.Next = Mdl;
}
- ReceiverRingCompletePacket(Ring, Packet);
+ ASSERT(IsZeroMemory(&Packet->ListEntry, sizeof (LIST_ENTRY)));
+ InsertTailList(List, &Packet->ListEntry);
+
return;
fail2:
static VOID
ReceiverRingProcessPacket(
IN PXENVIF_RECEIVER_RING Ring,
- IN PXENVIF_RECEIVER_PACKET Packet
+ IN PXENVIF_RECEIVER_PACKET Packet,
+ OUT PLIST_ENTRY List
)
{
PXENVIF_RECEIVER Receiver;
goto fail3;
if (Packet->MaximumSegmentSize != 0)
- ReceiverRingProcessLargePacket(Ring, Packet);
+ ReceiverRingProcessLargePacket(Ring, Packet, List);
else
- ReceiverRingProcessStandardPacket(Ring, Packet);
+ ReceiverRingProcessStandardPacket(Ring, Packet, List);
return;
1);
}
+static VOID
+ReceiverRingProcessPackets(
+ IN PXENVIF_RECEIVER_RING Ring,
+ OUT PLIST_ENTRY List,
+ OUT PULONG Count
+ )
+{
+ PLIST_ENTRY ListEntry;
+
+ while (!IsListEmpty(&Ring->PacketList)) {
+ PXENVIF_RECEIVER_PACKET Packet;
+
+ ListEntry = RemoveHeadList(&Ring->PacketList);
+ ASSERT3P(ListEntry, !=, &Ring->PacketList);
+
+ RtlZeroMemory(ListEntry, sizeof (LIST_ENTRY));
+
+ Packet = CONTAINING_RECORD(ListEntry, XENVIF_RECEIVER_PACKET, ListEntry);
+ ReceiverRingProcessPacket(Ring, Packet, List);
+ }
+
+ for (ListEntry = List->Flink;
+ ListEntry != List;
+ ListEntry = ListEntry->Flink) {
+ PXENVIF_RECEIVER_PACKET Packet;
+
+ Packet = CONTAINING_RECORD(ListEntry, XENVIF_RECEIVER_PACKET, ListEntry);
+
+ ReceiverRingProcessTag(Ring, Packet);
+ ReceiverRingProcessChecksum(Ring, Packet);
+
+ (*Count)++;
+ }
+}
+
+static FORCEINLINE VOID
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__ReceiverRingAcquireLock(
+ IN PXENVIF_RECEIVER_RING Ring
+ )
+{
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+
+ KeAcquireSpinLockAtDpcLevel(&Ring->Lock);
+}
+
+static DECLSPEC_NOINLINE VOID
+ReceiverRingAcquireLock(
+ IN PXENVIF_RECEIVER_RING Ring
+ )
+{
+ __ReceiverRingAcquireLock(Ring);
+}
+
static FORCEINLINE VOID
-__ReceiverRingSwizzle(
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__ReceiverRingReleaseLock(
IN PXENVIF_RECEIVER_RING Ring
)
{
PXENVIF_FRONTEND Frontend;
PXENVIF_VIF_CONTEXT Context;
LIST_ENTRY List;
- PLIST_ENTRY ListEntry;
+ ULONG Count;
+ BOOLEAN More;
+
+ ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
Receiver = Ring->Receiver;
Frontend = Receiver->Frontend;
Context = PdoGetVifContext(FrontendGetPdo(Frontend));
InitializeListHead(&List);
+ Count = 0;
- ListEntry = InterlockedExchangePointer(&Ring->PacketQueue, NULL);
-
- // Packets are held in the queue in reverse order so that the most
- // recent is always head of the list. This is necessary to allow
- // addition to the list to be done atomically.
-
- while (ListEntry != NULL) {
- PLIST_ENTRY NextEntry;
-
- NextEntry = ListEntry->Blink;
- ListEntry->Flink = ListEntry->Blink = ListEntry;
+ ReceiverRingProcessPackets(Ring, &List, &Count);
+ ASSERT(EQUIV(IsListEmpty(&List), Count == 0));
+ ASSERT(IsListEmpty(&Ring->PacketList));
- InsertHeadList(&List, ListEntry);
+ // We need to bump Loaned before dropping the lock to avoid VifDisable()
+ // returning prematurely.
+ __InterlockedAdd(&Receiver->Loaned, Count);
- ListEntry = NextEntry;
- }
-
- while (!IsListEmpty(&List)) {
- PXENVIF_RECEIVER_PACKET Packet;
-
- ListEntry = RemoveHeadList(&List);
- ASSERT3P(ListEntry, !=, &List);
+#pragma prefast(disable:26110)
+ KeReleaseSpinLockFromDpcLevel(&Ring->Lock);
- RtlZeroMemory(ListEntry, sizeof (LIST_ENTRY));
+ More = !IsListEmpty(&List) ? TRUE : FALSE;
- Packet = CONTAINING_RECORD(ListEntry, XENVIF_RECEIVER_PACKET, ListEntry);
- ReceiverRingProcessPacket(Ring, Packet);
- }
-
- while (!IsListEmpty(&Ring->PacketComplete)) {
+ while (More) {
+ PLIST_ENTRY ListEntry;
PXENVIF_RECEIVER_PACKET Packet;
PXENVIF_PACKET_INFO Info;
PUCHAR BaseVa;
PETHERNET_ADDRESS DestinationAddress;
ETHERNET_ADDRESS_TYPE Type;
- ListEntry = RemoveHeadList(&Ring->PacketComplete);
- ASSERT3P(ListEntry, !=, &Ring->PacketComplete);
+ ListEntry = RemoveHeadList(&List);
+ ASSERT3P(ListEntry, !=, &List);
RtlZeroMemory(ListEntry, sizeof (LIST_ENTRY));
+ ASSERT(More);
+ More = !IsListEmpty(&List) ? TRUE : FALSE;
+
Packet = CONTAINING_RECORD(ListEntry,
XENVIF_RECEIVER_PACKET,
ListEntry);
XENVIF_RECEIVER_UDP_PACKETS,
1);
- if (Packet->MaximumSegmentSize != 0)
+ if (Packet->MaximumSegmentSize != 0)
FrontendIncrementStatistic(Frontend,
XENVIF_RECEIVER_GSO_PACKETS,
1);
- if (Packet->Flags.IpChecksumSucceeded != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_IPV4_CHECKSUM_SUCCEEDED,
- 1);
-
- if (Packet->Flags.IpChecksumFailed != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_IPV4_CHECKSUM_FAILED,
- 1);
-
- if (Packet->Flags.IpChecksumNotValidated != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_IPV4_CHECKSUM_NOT_VALIDATED,
- 1);
-
- if (Packet->Flags.TcpChecksumSucceeded != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_TCP_CHECKSUM_SUCCEEDED,
- 1);
-
- if (Packet->Flags.TcpChecksumFailed != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_TCP_CHECKSUM_FAILED,
- 1);
-
- if (Packet->Flags.TcpChecksumNotValidated != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_TCP_CHECKSUM_NOT_VALIDATED,
- 1);
-
- if (Packet->Flags.UdpChecksumSucceeded != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_UDP_CHECKSUM_SUCCEEDED,
- 1);
-
- if (Packet->Flags.UdpChecksumFailed != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_UDP_CHECKSUM_FAILED,
- 1);
-
- if (Packet->Flags.UdpChecksumNotValidated != 0)
- FrontendIncrementStatistic(Frontend,
- XENVIF_RECEIVER_UDP_CHECKSUM_NOT_VALIDATED,
- 1);
-
- (VOID) InterlockedIncrement(&Receiver->Loaned);
+ if (Packet->Flags.IpChecksumSucceeded != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_IPV4_CHECKSUM_SUCCEEDED,
+ 1);
+
+ if (Packet->Flags.IpChecksumFailed != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_IPV4_CHECKSUM_FAILED,
+ 1);
+
+ if (Packet->Flags.IpChecksumNotValidated != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_IPV4_CHECKSUM_NOT_VALIDATED,
+ 1);
+
+ if (Packet->Flags.TcpChecksumSucceeded != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_TCP_CHECKSUM_SUCCEEDED,
+ 1);
+
+ if (Packet->Flags.TcpChecksumFailed != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_TCP_CHECKSUM_FAILED,
+ 1);
+
+ if (Packet->Flags.TcpChecksumNotValidated != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_TCP_CHECKSUM_NOT_VALIDATED,
+ 1);
+
+ if (Packet->Flags.UdpChecksumSucceeded != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_UDP_CHECKSUM_SUCCEEDED,
+ 1);
+
+ if (Packet->Flags.UdpChecksumFailed != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_UDP_CHECKSUM_FAILED,
+ 1);
+
+ if (Packet->Flags.UdpChecksumNotValidated != 0)
+ FrontendIncrementStatistic(Frontend,
+ XENVIF_RECEIVER_UDP_CHECKSUM_NOT_VALIDATED,
+ 1);
VifReceiverQueuePacket(Context,
Ring->Index,
Packet->TagControlInformation,
&Packet->Info,
&Packet->Hash,
- !IsListEmpty(&Ring->PacketComplete) ? TRUE : FALSE,
+ More,
Packet);
+
+ --Count;
}
+
+ ASSERT3U(Count, ==, 0);
}
-static FORCEINLINE VOID
-__drv_requiresIRQL(DISPATCH_LEVEL)
-__ReceiverRingAcquireLock(
+static DECLSPEC_NOINLINE VOID
+ReceiverRingReleaseLock(
IN PXENVIF_RECEIVER_RING Ring
)
{
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- KeAcquireSpinLockAtDpcLevel(&Ring->Lock);
+ __ReceiverRingReleaseLock(Ring);
}
-static DECLSPEC_NOINLINE VOID
-ReceiverRingAcquireLock(
+static FORCEINLINE VOID
+__ReceiverRingStop(
IN PXENVIF_RECEIVER_RING Ring
)
{
- __ReceiverRingAcquireLock(Ring);
+ Ring->Stopped = TRUE;
}
static FORCEINLINE VOID
-__drv_requiresIRQL(DISPATCH_LEVEL)
-__ReceiverRingReleaseLock(
+__ReceiverRingStart(
IN PXENVIF_RECEIVER_RING Ring
)
{
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
-#pragma prefast(disable:26110)
- KeReleaseSpinLockFromDpcLevel(&Ring->Lock);
+ Ring->Stopped = FALSE;
}
-static DECLSPEC_NOINLINE VOID
-ReceiverRingReleaseLock(
+static FORCEINLINE BOOLEAN
+__ReceiverRingIsStopped(
IN PXENVIF_RECEIVER_RING Ring
)
{
- __ReceiverRingReleaseLock(Ring);
+ return Ring->Stopped;
}
-__drv_functionClass(KDEFERRED_ROUTINE)
-__drv_maxIRQL(DISPATCH_LEVEL)
-__drv_minIRQL(PASSIVE_LEVEL)
-__drv_sameIRQL
-static VOID
-ReceiverRingDpc(
- IN PKDPC Dpc,
- IN PVOID Context,
- IN PVOID Argument1,
- IN PVOID Argument2
+static FORCEINLINE VOID
+__ReceiverRingTrigger(
+ IN PXENVIF_RECEIVER_RING Ring,
+ IN BOOLEAN Locked
)
{
- PXENVIF_RECEIVER_RING Ring = Context;
+ PXENVIF_RECEIVER Receiver;
- UNREFERENCED_PARAMETER(Dpc);
- UNREFERENCED_PARAMETER(Argument1);
- UNREFERENCED_PARAMETER(Argument2);
+ Receiver = Ring->Receiver;
- ASSERT(Ring != NULL);
+ if (!Locked)
+ __ReceiverRingAcquireLock(Ring);
- __ReceiverRingSwizzle(Ring);
-}
+ if (Ring->Connected)
+ (VOID) XENBUS_EVTCHN(Trigger,
+ &Receiver->EvtchnInterface,
+ Ring->Channel);
-static FORCEINLINE VOID
-__ReceiverRingStop(
- IN PXENVIF_RECEIVER_RING Ring
- )
-{
- Ring->Stopped = TRUE;
+ if (!Locked)
+ __ReceiverRingReleaseLock(Ring);
}
static FORCEINLINE VOID
-__ReceiverRingStart(
- IN PXENVIF_RECEIVER_RING Ring
+__ReceiverRingSend(
+ IN PXENVIF_RECEIVER_RING Ring,
+ IN BOOLEAN Locked
)
{
- Ring->Stopped = FALSE;
-}
+ PXENVIF_RECEIVER Receiver;
-static FORCEINLINE BOOLEAN
-__ReceiverRingIsStopped(
- IN PXENVIF_RECEIVER_RING Ring
- )
-{
- return Ring->Stopped;
+ Receiver = Ring->Receiver;
+
+ if (!Locked)
+ __ReceiverRingAcquireLock(Ring);
+
+ if (Ring->Connected)
+ (VOID) XENBUS_EVTCHN(Send,
+ &Receiver->EvtchnInterface,
+ Ring->Channel);
+
+ if (!Locked)
+ __ReceiverRingReleaseLock(Ring);
}
static FORCEINLINE VOID
IN BOOLEAN Locked
)
{
- PXENVIF_RECEIVER Receiver;
- PXENVIF_FRONTEND Frontend;
PMDL Mdl;
- Receiver = Ring->Receiver;
- Frontend = Receiver->Frontend;
-
Mdl = &Packet->Mdl;
while (Mdl != NULL) {
if (__ReceiverRingIsStopped(Ring)) {
__ReceiverRingStart(Ring);
- PollerTrigger(FrontendGetPoller(Frontend),
- Ring->Index,
- XENVIF_POLLER_EVENT_RECEIVE);
+ __ReceiverRingTrigger(Ring, TRUE);
}
if (!Locked)
#pragma warning (pop)
- if (Notify) {
- PXENVIF_RECEIVER Receiver;
- PXENVIF_FRONTEND Frontend;
-
- Receiver = Ring->Receiver;
- Frontend = Receiver->Frontend;
-
- PollerSend(FrontendGetPoller(Frontend),
- Ring->Index,
- XENVIF_POLLER_EVENT_RECEIVE);
- }
+ if (Notify)
+ __ReceiverRingSend(Ring, TRUE);
Ring->RequestsPushed = Ring->RequestsPosted;
}
(Ring->Enabled) ? "ENABLED" : "DISABLED",
(__ReceiverRingIsStopped(Ring)) ? "STOPPED" : "RUNNING");
- XENBUS_DEBUG(Printf,
- &Receiver->DebugInterface,
- "Dpcs = %lu\n",
- Ring->Dpcs);
-
// Dump front ring
XENBUS_DEBUG(Printf,
&Receiver->DebugInterface,
Ring->RequestsPosted,
Ring->RequestsPushed,
Ring->ResponsesProcessed);
-}
-
-static FORCEINLINE VOID
-__ReceiverRingQueuePacket(
- IN PXENVIF_RECEIVER_RING Ring,
- IN PXENVIF_RECEIVER_PACKET Packet
- )
-{
- PLIST_ENTRY ListEntry;
- PLIST_ENTRY Old;
- PLIST_ENTRY New;
-
- ListEntry = &Packet->ListEntry;
-
- do {
- Old = Ring->PacketQueue;
- ListEntry->Blink = Ring->PacketQueue;
- New = ListEntry;
- } while (InterlockedCompareExchangePointer(&Ring->PacketQueue, (PVOID)New, (PVOID)Old) != Old);
+ // Dump event channel
+ XENBUS_DEBUG(Printf,
+ &Receiver->DebugInterface,
+ "[%s]: Events = %lu Dpcs = %lu\n",
+ FrontendIsSplit(Frontend) ? "RX" : "COMBINED",
+ Ring->Events,
+ Ring->Dpcs);
}
static DECLSPEC_NOINLINE BOOLEAN
RING_IDX WorkToDo;
RING_FINAL_CHECK_FOR_RESPONSES(&Ring->Front, WorkToDo);
- if (!WorkToDo)
- break;
+ if (WorkToDo)
+ continue;
+
+ break;
}
while (rsp_cons != rsp_prod && !Retry) {
Packet->Flags.Value = flags;
ASSERT(IsZeroMemory(&Packet->ListEntry, sizeof (LIST_ENTRY)));
- __ReceiverRingQueuePacket(Ring, Packet);
+ InsertTailList(&Ring->PacketList, &Packet->ListEntry);
}
if (rsp_cons - Ring->Front.rsp_cons > XENVIF_RECEIVER_BATCH(Ring))
if (!__ReceiverRingIsStopped(Ring))
ReceiverRingFill(Ring);
- if (Ring->PacketQueue != NULL &&
- KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- Ring->Dpcs++;
-
done:
return Retry;
#undef XENVIF_RECEIVER_BATCH
}
+static FORCEINLINE VOID
+__ReceiverRingUnmask(
+ IN PXENVIF_RECEIVER_RING Ring
+ )
+{
+ PXENVIF_RECEIVER Receiver;
+
+ if (!Ring->Connected)
+ return;
+
+ Receiver = Ring->Receiver;
+
+ XENBUS_EVTCHN(Unmask,
+ &Receiver->EvtchnInterface,
+ Ring->Channel,
+ FALSE,
+ TRUE);
+}
+
+static FORCEINLINE BOOLEAN
+__ReceiverRingDpcTimeout(
+ IN PXENVIF_RECEIVER_RING Ring
+ )
+{
+ KDPC_WATCHDOG_INFORMATION Watchdog;
+ NTSTATUS status;
+
+ UNREFERENCED_PARAMETER(Ring);
+
+ RtlZeroMemory(&Watchdog, sizeof (Watchdog));
+
+ status = KeQueryDpcWatchdogInformation(&Watchdog);
+ ASSERT(NT_SUCCESS(status));
+
+ if (Watchdog.DpcTimeLimit == 0 ||
+ Watchdog.DpcWatchdogLimit == 0)
+ return FALSE;
+
+ if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) &&
+ Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2))
+ return FALSE;
+
+ return TRUE;
+}
+
#define TIME_US(_us) ((_us) * 10)
#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
#define TIME_S(_s) (TIME_MS((_s) * 1000))
#define TIME_RELATIVE(_t) (-(_t))
+__drv_functionClass(KDEFERRED_ROUTINE)
+__drv_maxIRQL(DISPATCH_LEVEL)
+__drv_minIRQL(DISPATCH_LEVEL)
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__drv_sameIRQL
+static VOID
+ReceiverRingDpc(
+ IN PKDPC Dpc,
+ IN PVOID Context,
+ IN PVOID Argument1,
+ IN PVOID Argument2
+ )
+{
+ PXENVIF_RECEIVER_RING Ring = Context;
+
+ UNREFERENCED_PARAMETER(Dpc);
+ UNREFERENCED_PARAMETER(Argument1);
+ UNREFERENCED_PARAMETER(Argument2);
+
+ ASSERT(Ring != NULL);
+
+ for (;;) {
+ BOOLEAN Retry;
+
+ __ReceiverRingAcquireLock(Ring);
+ Retry = ReceiverRingPoll(Ring);
+ __ReceiverRingReleaseLock(Ring);
+
+ if (!Retry) {
+ __ReceiverRingUnmask(Ring);
+ break;
+ }
+
+ if (__ReceiverRingDpcTimeout(Ring)) {
+ LARGE_INTEGER Delay;
+
+ Delay.QuadPart = TIME_RELATIVE(TIME_US(100));
+
+ KeSetTimer(&Ring->Timer, Delay, &Ring->TimerDpc);
+ break;
+ }
+ }
+}
+
+KSERVICE_ROUTINE ReceiverRingEvtchnCallback;
+
+BOOLEAN
+ReceiverRingEvtchnCallback(
+ IN PKINTERRUPT InterruptObject,
+ IN PVOID Argument
+ )
+{
+ PXENVIF_RECEIVER_RING Ring = Argument;
+ PXENVIF_RECEIVER Receiver;
+ PXENVIF_FRONTEND Frontend;
+
+ UNREFERENCED_PARAMETER(InterruptObject);
+
+ ASSERT(Ring != NULL);
+
+ Ring->Events++;
+
+ if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
+ Ring->Dpcs++;
+
+ Receiver = Ring->Receiver;
+ Frontend = Receiver->Frontend;
+
+ if (!FrontendIsSplit(Frontend))
+ TransmitterNotify(FrontendGetTransmitter(Frontend),
+ Ring->Index);
+
+ return TRUE;
+}
+
#define XENVIF_RECEIVER_WATCHDOG_PERIOD 30
static NTSTATUS
if (Ring->Shared->rsp_prod != rsp_prod &&
Ring->Front.rsp_cons == rsp_cons) {
PXENVIF_RECEIVER Receiver;
- PXENVIF_FRONTEND Frontend;
Receiver = Ring->Receiver;
- Frontend = Receiver->Frontend;
XENBUS_DEBUG(Trigger,
&Receiver->DebugInterface,
Ring->DebugCallback);
// Try to move things along
- PollerTrigger(FrontendGetPoller(Frontend),
- Ring->Index,
- XENVIF_POLLER_EVENT_RECEIVE);
- PollerSend(FrontendGetPoller(Frontend),
- Ring->Index,
- XENVIF_POLLER_EVENT_RECEIVE);
+ __ReceiverRingTrigger(Ring, TRUE);
+ __ReceiverRingSend(Ring, TRUE);
}
KeMemoryBarrier();
if ((*Ring)->Path == NULL)
goto fail2;
- InitializeListHead(&(*Ring)->PacketComplete);
+ InitializeListHead(&(*Ring)->PacketList);
+
+ KeInitializeDpc(&(*Ring)->Dpc, ReceiverRingDpc, *Ring);
+ KeInitializeTimer(&(*Ring)->Timer);
+ KeInitializeDpc(&(*Ring)->TimerDpc, ReceiverRingDpc, *Ring);
status = RtlStringCbPrintfA(Name,
sizeof (Name),
if (!NT_SUCCESS(status))
goto fail7;
- KeInitializeThreadedDpc(&(*Ring)->Dpc, ReceiverRingDpc, *Ring);
-
return STATUS_SUCCESS;
fail7:
fail3:
Error("fail3\n");
- RtlZeroMemory(&(*Ring)->PacketComplete, sizeof (LIST_ENTRY));
+ RtlZeroMemory(&(*Ring)->TimerDpc, sizeof (KDPC));
+ RtlZeroMemory(&(*Ring)->Timer, sizeof (KTIMER));
+ RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC));
+
+ RtlZeroMemory(&(*Ring)->PacketList, sizeof (LIST_ENTRY));
FrontendFreePath(Frontend, (*Ring)->Path);
(*Ring)->Path = NULL;
ASSERT(!Ring->Connected);
+ Ring->Channel = XENBUS_EVTCHN(Open,
+ &Receiver->EvtchnInterface,
+ XENBUS_EVTCHN_TYPE_UNBOUND,
+ ReceiverRingEvtchnCallback,
+ Ring,
+ FrontendGetBackendDomain(Frontend),
+ TRUE);
+
+ status = STATUS_UNSUCCESSFUL;
+ if (Ring->Channel == NULL)
+ goto fail6;
+
+ status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber);
+ ASSERT(NT_SUCCESS(status));
+
+ KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber);
+ KeSetTargetProcessorDpcEx(&Ring->TimerDpc, &ProcNumber);
+
+ (VOID) XENBUS_EVTCHN(Bind,
+ &Receiver->EvtchnInterface,
+ Ring->Channel,
+ ProcNumber.Group,
+ ProcNumber.Number);
+
+ (VOID) XENBUS_EVTCHN(Unmask,
+ &Receiver->EvtchnInterface,
+ Ring->Channel,
+ FALSE,
+ TRUE);
+
Ring->Connected = TRUE;
status = XENBUS_DEBUG(Register,
Ring,
&Ring->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail6;
+ goto fail7;
- status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber);
- ASSERT(NT_SUCCESS(status));
+ return STATUS_SUCCESS;
- KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber);
+fail7:
+ Error("fail7\n");
- return STATUS_SUCCESS;
+ Ring->Connected = FALSE;
+
+ XENBUS_EVTCHN(Close,
+ &Receiver->EvtchnInterface,
+ Ring->Channel);
+ Ring->Channel = NULL;
+
+ Ring->Events = 0;
fail6:
Error("fail6\n");
- Ring->Connected = FALSE;
-
fail5:
Error("fail5\n");
{
PXENVIF_RECEIVER Receiver;
PXENVIF_FRONTEND Frontend;
+ ULONG Port;
PCHAR Path;
NTSTATUS status;
if (!NT_SUCCESS(status))
goto fail1;
+ Port = XENBUS_EVTCHN(GetPort,
+ &Receiver->EvtchnInterface,
+ Ring->Channel);
+
+ status = XENBUS_STORE(Printf,
+ &Receiver->StoreInterface,
+ Transaction,
+ Path,
+ FrontendIsSplit(Frontend) ? "event-channel-rx" : "event-channel",
+ "%u",
+ Port);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
return STATUS_SUCCESS;
+fail2:
+ Error("fail2\n");
+
fail1:
Error("fail1 (%08x)\n", status);
Ring->Enabled = TRUE;
+ (VOID) KeInsertQueueDpc(&Ring->Dpc, NULL, NULL);
+
__ReceiverRingReleaseLock(Ring);
Info("%s[%u]: <====\n",
Ring->Enabled = FALSE;
Ring->Stopped = FALSE;
- if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
- Ring->Dpcs++;
-
__ReceiverRingReleaseLock(Ring);
+ //
+ // No new timers can be scheduled once Enabled goes to FALSE.
+ // Cancel any existing ones.
+ //
+ (VOID) KeCancelTimer(&Ring->Timer);
+
Info("%s[%u]: <====\n",
FrontendGetPath(Frontend),
Ring->Index);
Receiver = Ring->Receiver;
Frontend = Receiver->Frontend;
- Ring->Dpcs = 0;
-
__ReceiverRingEmpty(Ring);
ASSERT(Ring->Connected);
Ring->Connected = FALSE;
+ XENBUS_EVTCHN(Close,
+ &Receiver->EvtchnInterface,
+ Ring->Channel);
+ Ring->Channel = NULL;
+
+ Ring->Events = 0;
+ Ring->Dpcs = 0;
+
ASSERT3U(Ring->ResponsesProcessed, ==, Ring->RequestsPushed);
ASSERT3U(Ring->RequestsPushed, ==, Ring->RequestsPosted);
Frontend = Receiver->Frontend;
RtlZeroMemory(&Ring->Hash, sizeof (XENVIF_RECEIVER_HASH));
+ RtlZeroMemory(&Ring->TimerDpc, sizeof (KDPC));
+ RtlZeroMemory(&Ring->Timer, sizeof (KTIMER));
+ RtlZeroMemory(&Ring->Dpc, sizeof (KDPC));
Ring->BackfillSize = 0;
Ring->OffloadOptions.Value = 0;
- KeFlushQueuedDpcs();
- RtlZeroMemory(&Ring->Dpc, sizeof (KDPC));
-
ThreadAlert(Ring->WatchdogThread);
ThreadJoin(Ring->WatchdogThread);
Ring->WatchdogThread = NULL;
Ring->PacketCache);
Ring->PacketCache = NULL;
- ASSERT(IsListEmpty(&Ring->PacketComplete));
- RtlZeroMemory(&Ring->PacketComplete, sizeof (LIST_ENTRY));
+ ASSERT(IsListEmpty(&Ring->PacketList));
+ RtlZeroMemory(&Ring->PacketList, sizeof (LIST_ENTRY));
FrontendFreePath(Frontend, Ring->Path);
Ring->Path = NULL;
FdoGetGnttabInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
&(*Receiver)->GnttabInterface);
+ FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
+ &(*Receiver)->EvtchnInterface);
+
(*Receiver)->Frontend = Frontend;
status = XENBUS_CACHE(Acquire, &(*Receiver)->CacheInterface);
(*Receiver)->Frontend = NULL;
+ RtlZeroMemory(&(*Receiver)->EvtchnInterface,
+ sizeof (XENBUS_EVTCHN_INTERFACE));
+
RtlZeroMemory(&(*Receiver)->GnttabInterface,
sizeof (XENBUS_GNTTAB_INTERFACE));
if (!NT_SUCCESS(status))
goto fail2;
- status = XENBUS_GNTTAB(Acquire, &Receiver->GnttabInterface);
+ status = XENBUS_EVTCHN(Acquire, &Receiver->EvtchnInterface);
if (!NT_SUCCESS(status))
goto fail3;
+ status = XENBUS_GNTTAB(Acquire, &Receiver->GnttabInterface);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+
Index = 0;
while (Index < (LONG)FrontendGetNumQueues(Frontend)) {
PXENVIF_RECEIVER_RING Ring = Receiver->Ring[Index];
status = __ReceiverRingConnect(Ring);
if (!NT_SUCCESS(status))
- goto fail4;
+ goto fail5;
Index++;
}
Receiver,
&Receiver->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail6;
Trace("<====\n");
return STATUS_SUCCESS;
-fail5:
- Error("fail5\n");
+fail6:
+ Error("fail6\n");
Index = FrontendGetNumQueues(Frontend);
-fail4:
- Error("fail4\n");
+fail5:
+ Error("fail5\n");
while (--Index >= 0) {
PXENVIF_RECEIVER_RING Ring = Receiver->Ring[Index];
XENBUS_GNTTAB(Release, &Receiver->GnttabInterface);
+fail4:
+ Error("fail4\n");
+
+ XENBUS_EVTCHN(Release, &Receiver->EvtchnInterface);
+
fail3:
Error("fail3\n");
return status;
}
-BOOLEAN
-ReceiverPoll(
- IN PXENVIF_RECEIVER Receiver,
- IN ULONG Index
- )
-{
- PXENVIF_FRONTEND Frontend;
- ULONG NumQueues;
- PXENVIF_RECEIVER_RING Ring;
- BOOLEAN Retry;
-
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- Frontend = Receiver->Frontend;
-
- NumQueues = FrontendGetNumQueues(Frontend);
- if (Index >= NumQueues)
- return FALSE;
-
- Ring = Receiver->Ring[Index];
-
- __ReceiverRingAcquireLock(Ring);
- Retry = ReceiverRingPoll(Ring);
- __ReceiverRingReleaseLock(Ring);
-
- return Retry;
-}
-
VOID
ReceiverDisable(
IN PXENVIF_RECEIVER Receiver
XENBUS_GNTTAB(Release, &Receiver->GnttabInterface);
+ XENBUS_EVTCHN(Release, &Receiver->EvtchnInterface);
+
XENBUS_STORE(Release, &Receiver->StoreInterface);
XENBUS_DEBUG(Release, &Receiver->DebugInterface);
Frontend = Receiver->Frontend;
+ ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
+ KeFlushQueuedDpcs();
+
ASSERT3U(Receiver->Returned, ==, Receiver->Loaned);
Receiver->Loaned = 0;
Receiver->Returned = 0;
Receiver->Frontend = NULL;
+ RtlZeroMemory(&Receiver->EvtchnInterface,
+ sizeof (XENBUS_EVTCHN_INTERFACE));
+
RtlZeroMemory(&Receiver->GnttabInterface,
sizeof (XENBUS_GNTTAB_INTERFACE));
LARGE_INTEGER Timeout;
ASSERT3U(KeGetCurrentIrql(), <, DISPATCH_LEVEL);
- KeFlushQueuedDpcs();
Frontend = Receiver->Frontend;
Trace("%s: ====>\n", FrontendGetPath(Frontend));
Returned = Receiver->Returned;
+
+ // Make sure Loaned is not sampled before Returned
+ KeMemoryBarrier();
+
Loaned = Receiver->Loaned;
ASSERT3S(Loaned - Returned, >=, 0);
Trace("%s: <====\n", FrontendGetPath(Frontend));
}
+VOID
+ReceiverTrigger(
+ IN PXENVIF_RECEIVER Receiver,
+ IN ULONG Index
+ )
+{
+ PXENVIF_RECEIVER_RING Ring;
+
+ Ring = Receiver->Ring[Index];
+
+ __ReceiverRingTrigger(Ring, FALSE);
+}
+
+VOID
+ReceiverSend(
+ IN PXENVIF_RECEIVER Receiver,
+ IN ULONG Index
+ )
+{
+ PXENVIF_RECEIVER_RING Ring;
+
+ Ring = Receiver->Ring[Index];
+
+ __ReceiverRingSend(Ring, FALSE);
+}
+
NTSTATUS
ReceiverSetHashAlgorithm(
IN PXENVIF_RECEIVER Receiver,
IN PXENVIF_RECEIVER Receiver
);
-extern BOOLEAN
-ReceiverPoll(
- IN PXENVIF_RECEIVER Receiver,
- IN ULONG Index
- );
-
extern VOID
ReceiverDisable(
IN PXENVIF_RECEIVER Receiver
#include <cache_interface.h>
#include <gnttab_interface.h>
#include <range_set_interface.h>
+#include <evtchn_interface.h>
#include "pdo.h"
#include "frontend.h"
netif_tx_front_ring_t Front;
netif_tx_sring_t *Shared;
PXENBUS_GNTTAB_ENTRY Entry;
+ PXENBUS_EVTCHN_CHANNEL Channel;
+ KDPC Dpc;
+ ULONG Dpcs;
+ KTIMER Timer;
+ KDPC TimerDpc;
+ ULONG Events;
BOOLEAN Connected;
BOOLEAN Enabled;
BOOLEAN Stopped;
XENBUS_CACHE_INTERFACE CacheInterface;
XENBUS_GNTTAB_INTERFACE GnttabInterface;
XENBUS_RANGE_SET_INTERFACE RangeSetInterface;
+ XENBUS_EVTCHN_INTERFACE EvtchnInterface;
PXENVIF_TRANSMITTER_RING *Ring;
BOOLEAN MulticastControl;
ULONG DisableIpVersion4Gso;
Ring->PacketsUnprepared,
Ring->PacketsSent,
Ring->PacketsCompleted);
+
+ if (FrontendIsSplit(Frontend)) {
+ // Dump event channel
+ XENBUS_DEBUG(Printf,
+ &Transmitter->DebugInterface,
+ "Events = %lu Dpcs = %lu\n",
+ Ring->Events,
+ Ring->Dpcs);
+ }
}
static BOOLEAN
RING_IDX rsp_cons;
ULONG Extra;
+ if (Retry)
+ break;
+
KeMemoryBarrier();
rsp_prod = Ring->Shared->rsp_prod;
KeMemoryBarrier();
- if (rsp_cons == rsp_prod || Retry)
+ if (rsp_cons == rsp_prod) {
+ RING_IDX WorkToDo;
+
+ RING_FINAL_CHECK_FOR_RESPONSES(&Ring->Front, WorkToDo);
+ if (WorkToDo)
+ continue;
+
break;
+ }
Extra = 0;
while (rsp_cons != rsp_prod && !Retry) {
KeMemoryBarrier();
Ring->Front.rsp_cons = rsp_cons;
- Ring->Shared->rsp_event = rsp_cons + 1;
}
done:
}
static FORCEINLINE VOID
-__TransmitterRingPushRequests(
+__TransmitterRingTrigger(
+ IN PXENVIF_TRANSMITTER_RING Ring
+ )
+{
+ PXENVIF_TRANSMITTER Transmitter;
+ PXENVIF_FRONTEND Frontend;
+
+ Transmitter = Ring->Transmitter;
+ Frontend = Transmitter->Frontend;
+
+ if (!Ring->Connected)
+ return;
+
+ if (FrontendIsSplit(Frontend)) {
+ ASSERT(Ring->Channel != NULL);
+
+ (VOID) XENBUS_EVTCHN(Trigger,
+ &Transmitter->EvtchnInterface,
+ Ring->Channel);
+ } else {
+ ReceiverTrigger(FrontendGetReceiver(Frontend),
+ Ring->Index);
+ }
+}
+
+static FORCEINLINE VOID
+__TransmitterRingSend(
IN PXENVIF_TRANSMITTER_RING Ring
)
{
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
- BOOLEAN Notify;
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
+ if (!Ring->Connected)
+ return;
+
+ if (FrontendIsSplit(Frontend)) {
+ ASSERT(Ring->Channel != NULL);
+
+ (VOID) XENBUS_EVTCHN(Send,
+ &Transmitter->EvtchnInterface,
+ Ring->Channel);
+ } else {
+ ReceiverSend(FrontendGetReceiver(Frontend),
+ Ring->Index);
+ }
+}
+
+static FORCEINLINE VOID
+__TransmitterRingPushRequests(
+ IN PXENVIF_TRANSMITTER_RING Ring
+ )
+{
+ BOOLEAN Notify;
+
if (Ring->RequestsPosted == Ring->RequestsPushed)
return;
#pragma warning (pop)
if (Notify)
- PollerSend(FrontendGetPoller(Frontend),
- Ring->Index,
- XENVIF_POLLER_EVENT_TRANSMIT);
+ __TransmitterRingSend(Ring);
Ring->RequestsPushed = Ring->RequestsPosted;
}
__TransmitterRingReleaseLock(Ring);
}
+static FORCEINLINE VOID
+__TransmitterRingUnmask(
+ IN PXENVIF_TRANSMITTER_RING Ring
+ )
+{
+ PXENVIF_TRANSMITTER Transmitter;
+ PXENVIF_FRONTEND Frontend;
+
+ Transmitter = Ring->Transmitter;
+ Frontend = Transmitter->Frontend;
+
+ if (!Ring->Connected || !FrontendIsSplit(Frontend))
+ return;
+
+ XENBUS_EVTCHN(Unmask,
+ &Transmitter->EvtchnInterface,
+ Ring->Channel,
+ FALSE,
+ TRUE);
+}
+
+static FORCEINLINE BOOLEAN
+__TransmitterRingDpcTimeout(
+ IN PXENVIF_TRANSMITTER_RING Ring
+ )
+{
+ KDPC_WATCHDOG_INFORMATION Watchdog;
+ NTSTATUS status;
+
+ UNREFERENCED_PARAMETER(Ring);
+
+ RtlZeroMemory(&Watchdog, sizeof (Watchdog));
+
+ status = KeQueryDpcWatchdogInformation(&Watchdog);
+ ASSERT(NT_SUCCESS(status));
+
+ if (Watchdog.DpcTimeLimit == 0 ||
+ Watchdog.DpcWatchdogLimit == 0)
+ return FALSE;
+
+ if (Watchdog.DpcTimeCount > (Watchdog.DpcTimeLimit / 2) &&
+ Watchdog.DpcWatchdogCount > (Watchdog.DpcWatchdogLimit / 2))
+ return FALSE;
+
+ return TRUE;
+}
+
#define TIME_US(_us) ((_us) * 10)
#define TIME_MS(_ms) (TIME_US((_ms) * 1000))
#define TIME_S(_s) (TIME_MS((_s) * 1000))
#define TIME_RELATIVE(_t) (-(_t))
+__drv_functionClass(KDEFERRED_ROUTINE)
+__drv_maxIRQL(DISPATCH_LEVEL)
+__drv_minIRQL(DISPATCH_LEVEL)
+__drv_requiresIRQL(DISPATCH_LEVEL)
+__drv_sameIRQL
+static VOID
+TransmitterRingDpc(
+ IN PKDPC Dpc,
+ IN PVOID Context,
+ IN PVOID Argument1,
+ IN PVOID Argument2
+ )
+{
+ PXENVIF_TRANSMITTER_RING Ring = Context;
+
+ UNREFERENCED_PARAMETER(Dpc);
+ UNREFERENCED_PARAMETER(Argument1);
+ UNREFERENCED_PARAMETER(Argument2);
+
+ ASSERT(Ring != NULL);
+
+ for (;;) {
+ BOOLEAN Retry;
+
+ __TransmitterRingAcquireLock(Ring);
+ Retry = TransmitterRingPoll(Ring);
+ __TransmitterRingReleaseLock(Ring);
+
+ if (!Retry) {
+ __TransmitterRingUnmask(Ring);
+ break;
+ }
+
+ if (__TransmitterRingDpcTimeout(Ring)) {
+ LARGE_INTEGER Delay;
+
+ Delay.QuadPart = TIME_RELATIVE(TIME_US(100));
+
+ KeSetTimer(&Ring->Timer, Delay, &Ring->TimerDpc);
+ break;
+ }
+ }
+}
+
+KSERVICE_ROUTINE TransmitterRingEvtchnCallback;
+
+BOOLEAN
+TransmitterRingEvtchnCallback(
+ IN PKINTERRUPT InterruptObject,
+ IN PVOID Argument
+ )
+{
+ PXENVIF_TRANSMITTER_RING Ring = Argument;
+ PXENVIF_TRANSMITTER Transmitter;
+ PXENVIF_FRONTEND Frontend;
+
+ UNREFERENCED_PARAMETER(InterruptObject);
+
+ ASSERT(Ring != NULL);
+
+ Transmitter = Ring->Transmitter;
+ Frontend = Transmitter->Frontend;
+
+ ASSERT(FrontendIsSplit(Frontend));
+
+ Ring->Events++;
+
+ if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
+ Ring->Dpcs++;
+
+ return TRUE;
+}
+
#define XENVIF_TRANSMITTER_WATCHDOG_PERIOD 30
static NTSTATUS
if (Ring->PacketsQueued == PacketsQueued &&
Ring->PacketsCompleted != PacketsQueued) {
PXENVIF_TRANSMITTER Transmitter;
- PXENVIF_FRONTEND Frontend;
Transmitter = Ring->Transmitter;
- Frontend = Transmitter->Frontend;
XENBUS_DEBUG(Trigger,
&Transmitter->DebugInterface,
Ring->DebugCallback);
// Try to move things along
- PollerTrigger(FrontendGetPoller(Frontend),
- Ring->Index,
- XENVIF_POLLER_EVENT_TRANSMIT);
- PollerSend(FrontendGetPoller(Frontend),
- Ring->Index,
- XENVIF_POLLER_EVENT_TRANSMIT);
+ __TransmitterRingTrigger(Ring);
+ __TransmitterRingSend(Ring);
}
PacketsQueued = Ring->PacketsQueued;
InitializeListHead(&(*Ring)->RequestQueue);
InitializeListHead(&(*Ring)->PacketComplete);
+ KeInitializeDpc(&(*Ring)->Dpc, TransmitterRingDpc, *Ring);
+ KeInitializeTimer(&(*Ring)->Timer);
+ KeInitializeDpc(&(*Ring)->TimerDpc, TransmitterRingDpc, *Ring);
+
status = RtlStringCbPrintfA(Name,
sizeof (Name),
"%s_transmitter_buffer",
fail3:
Error("fail3\n");
+ RtlZeroMemory(&(*Ring)->Dpc, sizeof (KDPC));
+
RtlZeroMemory(&(*Ring)->PacketComplete, sizeof (LIST_ENTRY));
RtlZeroMemory(&(*Ring)->RequestQueue, sizeof (LIST_ENTRY));
RtlZeroMemory(&(*Ring)->PacketQueue, sizeof (LIST_ENTRY));
PFN_NUMBER Pfn;
CHAR Name[MAXNAMELEN];
ULONG Index;
+ PROCESSOR_NUMBER ProcNumber;
NTSTATUS status;
ASSERT(!Ring->Connected);
ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
+ if (FrontendIsSplit(Frontend)) {
+ Ring->Channel = XENBUS_EVTCHN(Open,
+ &Transmitter->EvtchnInterface,
+ XENBUS_EVTCHN_TYPE_UNBOUND,
+ TransmitterRingEvtchnCallback,
+ Ring,
+ FrontendGetBackendDomain(Frontend),
+ TRUE);
+
+ status = STATUS_UNSUCCESSFUL;
+ if (Ring->Channel == NULL)
+ goto fail6;
+
+ status = KeGetProcessorNumberFromIndex(Ring->Index, &ProcNumber);
+ ASSERT(NT_SUCCESS(status));
+
+ KeSetTargetProcessorDpcEx(&Ring->Dpc, &ProcNumber);
+ KeSetTargetProcessorDpcEx(&Ring->TimerDpc, &ProcNumber);
+
+ (VOID) XENBUS_EVTCHN(Bind,
+ &Transmitter->EvtchnInterface,
+ Ring->Channel,
+ ProcNumber.Group,
+ ProcNumber.Number);
+
+ (VOID) XENBUS_EVTCHN(Unmask,
+ &Transmitter->EvtchnInterface,
+ Ring->Channel,
+ FALSE,
+ TRUE);
+ }
+
status = XENBUS_DEBUG(Register,
&Transmitter->DebugInterface,
Name,
Ring,
&Ring->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail6;
+ goto fail7;
Ring->Connected = TRUE;
return STATUS_SUCCESS;
+fail7:
+ Error("fail7\n");
+
+ XENBUS_EVTCHN(Close,
+ &Transmitter->EvtchnInterface,
+ Ring->Channel);
+ Ring->Channel = NULL;
+
+ Ring->Events = 0;
+
fail6:
Error("fail6\n");
{
PXENVIF_TRANSMITTER Transmitter;
PXENVIF_FRONTEND Frontend;
+ ULONG Port;
PCHAR Path;
NTSTATUS status;
if (!NT_SUCCESS(status))
goto fail1;
+ if (!FrontendIsSplit(Frontend))
+ goto done;
+
+ Port = XENBUS_EVTCHN(GetPort,
+ &Transmitter->EvtchnInterface,
+ Ring->Channel);
+
+ status = XENBUS_STORE(Printf,
+ &Transmitter->StoreInterface,
+ Transaction,
+ Path,
+ "event-channel-tx",
+ "%u",
+ Port);
+ if (!NT_SUCCESS(status))
+ goto fail2;
+
+done:
return STATUS_SUCCESS;
+fail2:
+ Error("fail2\n");
+
fail1:
Error("fail1 (%08x)\n", status);
ASSERT(!Ring->Enabled);
Ring->Enabled = TRUE;
+ KeInsertQueueDpc(&Ring->Dpc, NULL, NULL);
+
__TransmitterRingReleaseLock(Ring);
Info("%s[%u]: <====\n",
ASSERT(Attempt < 100);
// Try to move things along
- PollerSend(FrontendGetPoller(Frontend),
- Ring->Index,
- XENVIF_POLLER_EVENT_TRANSMIT);
+ __TransmitterRingSend(Ring);
(VOID) TransmitterRingPoll(Ring);
if (State != XenbusStateConnected)
__TransmitterRingReleaseLock(Ring);
+ //
+ // No new timers can be scheduled once Enabled goes to FALSE.
+ // Cancel any existing ones.
+ //
+ (VOID) KeCancelTimer(&Ring->Timer);
+
Info("%s[%u]: <====\n",
FrontendGetPath(Frontend),
Ring->Index);
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
+ if (Ring->Channel != NULL) {
+ XENBUS_EVTCHN(Close,
+ &Transmitter->EvtchnInterface,
+ Ring->Channel);
+ Ring->Channel = NULL;
+
+ Ring->Events = 0;
+ }
+
+ Ring->Dpcs = 0;
+
ASSERT3U(Ring->ResponsesProcessed, ==, Ring->RequestsPushed);
ASSERT3U(Ring->RequestsPushed, ==, Ring->RequestsPosted);
Transmitter = Ring->Transmitter;
Frontend = Transmitter->Frontend;
+ Ring->Dpcs = 0;
+
+ RtlZeroMemory(&Ring->TimerDpc, sizeof (KDPC));
+ RtlZeroMemory(&Ring->Timer, sizeof (KTIMER));
+ RtlZeroMemory(&Ring->Dpc, sizeof (KDPC));
+
ASSERT3U(Ring->PacketsCompleted, ==, Ring->PacketsSent);
ASSERT3U(Ring->PacketsSent, ==, Ring->PacketsPrepared - Ring->PacketsUnprepared);
ASSERT3U(Ring->PacketsPrepared, ==, Ring->PacketsCopied + Ring->PacketsGranted + Ring->PacketsFaked);
FdoGetGnttabInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
&(*Transmitter)->GnttabInterface);
+ FdoGetEvtchnInterface(PdoGetFdo(FrontendGetPdo(Frontend)),
+ &(*Transmitter)->EvtchnInterface);
+
(*Transmitter)->Frontend = Frontend;
KeInitializeSpinLock(&(*Transmitter)->Lock);
if (!NT_SUCCESS(status))
goto fail2;
- status = XENBUS_GNTTAB(Acquire, &Transmitter->GnttabInterface);
+ status = XENBUS_EVTCHN(Acquire, &Transmitter->EvtchnInterface);
if (!NT_SUCCESS(status))
goto fail3;
+ status = XENBUS_GNTTAB(Acquire, &Transmitter->GnttabInterface);
+ if (!NT_SUCCESS(status))
+ goto fail4;
+
if (Transmitter->DisableMulticastControl == 0) {
status = XENBUS_STORE(Read,
&Transmitter->StoreInterface,
status = __TransmitterRingConnect(Ring);
if (!NT_SUCCESS(status))
- goto fail4;
+ goto fail5;
Index++;
}
Transmitter,
&Transmitter->DebugCallback);
if (!NT_SUCCESS(status))
- goto fail5;
+ goto fail6;
Trace("<====\n");
return STATUS_SUCCESS;
-fail5:
- Error("fail5\n");
+fail6:
+ Error("fail6\n");
Index = FrontendGetNumQueues(Frontend);
-fail4:
- Error("fail4\n");
+fail5:
+ Error("fail5\n");
while (--Index >= 0) {
PXENVIF_TRANSMITTER_RING Ring;
XENBUS_GNTTAB(Release, &Transmitter->GnttabInterface);
+fail4:
+ Error("fail4\n");
+
+ XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface);
+
fail3:
Error("fail3\n");
return STATUS_SUCCESS;
}
-BOOLEAN
-TransmitterPoll(
- IN PXENVIF_TRANSMITTER Transmitter,
- IN ULONG Index
- )
-{
- PXENVIF_FRONTEND Frontend;
- ULONG NumQueues;
- PXENVIF_TRANSMITTER_RING Ring;
- BOOLEAN Retry;
-
- ASSERT3U(KeGetCurrentIrql(), ==, DISPATCH_LEVEL);
-
- Frontend = Transmitter->Frontend;
-
- NumQueues = FrontendGetNumQueues(Frontend);
- if (Index >= NumQueues)
- return FALSE;
-
- Ring = Transmitter->Ring[Index];
-
- __TransmitterRingAcquireLock(Ring);
- Retry = TransmitterRingPoll(Ring);
- __TransmitterRingReleaseLock(Ring);
-
- return Retry;
-}
-
VOID
TransmitterDisable(
IN PXENVIF_TRANSMITTER Transmitter
XENBUS_GNTTAB(Release, &Transmitter->GnttabInterface);
+ XENBUS_EVTCHN(Release, &Transmitter->EvtchnInterface);
+
XENBUS_STORE(Release, &Transmitter->StoreInterface);
XENBUS_DEBUG(Release, &Transmitter->DebugInterface);
Frontend = Transmitter->Frontend;
ASSERT3U(KeGetCurrentIrql(), ==, PASSIVE_LEVEL);
+ KeFlushQueuedDpcs();
Index = FrontendGetMaxQueues(Frontend);
while (--Index >= 0) {
RtlZeroMemory(&Transmitter->DebugInterface,
sizeof (XENBUS_DEBUG_INTERFACE));
+ RtlZeroMemory(&Transmitter->EvtchnInterface,
+ sizeof (XENBUS_EVTCHN_INTERFACE));
+
Transmitter->DisableIpVersion4Gso = 0;
Transmitter->DisableIpVersion6Gso = 0;
Transmitter->AlwaysCopy = 0;
*Size = XENVIF_TRANSMITTER_RING_SIZE;
}
+VOID
+TransmitterNotify(
+ IN PXENVIF_TRANSMITTER Transmitter,
+ IN ULONG Index
+ )
+{
+ PXENVIF_FRONTEND Frontend;
+ PXENVIF_TRANSMITTER_RING Ring;
+
+ Frontend = Transmitter->Frontend;
+
+ ASSERT(!FrontendIsSplit(Frontend));
+
+ Ring = Transmitter->Ring[Index];
+
+ if (KeInsertQueueDpc(&Ring->Dpc, NULL, NULL))
+ Ring->Dpcs++;
+}
+
VOID
TransmitterQueryOffloadOptions(
IN PXENVIF_TRANSMITTER Transmitter,
IN PXENVIF_TRANSMITTER Transmitter
);
-extern BOOLEAN
-TransmitterPoll(
- IN PXENVIF_TRANSMITTER Transmitter,
- IN ULONG Index
- );
-
extern VOID
TransmitterDisable(
IN PXENVIF_TRANSMITTER Transmitter
Hash,
More,
Cookie);
-
}
VOID
IN PVOID Cookie
)
{
- KIRQL Irql;
-
- KeRaiseIrql(DISPATCH_LEVEL, &Irql);
-
switch (Context->Version) {
case 6:
__VifReceiverQueuePacketVersion6(Context,
ASSERT(FALSE);
break;
}
-
- KeLowerIrql(Irql);
}
VOID
<ClCompile Include="../../src/xenvif/mac.c" />
<ClCompile Include="../../src/xenvif/parse.c" />
<ClCompile Include="../../src/xenvif/pdo.c" />
- <ClCompile Include="../../src/xenvif/poller.c" />
<ClCompile Include="../../src/xenvif/receiver.c" />
<ClCompile Include="../../src/xenvif/registry.c" />
<ClCompile Include="../../src/xenvif/settings.c" />
<ClCompile Include="../../src/xenvif/mac.c" />
<ClCompile Include="../../src/xenvif/parse.c" />
<ClCompile Include="../../src/xenvif/pdo.c" />
- <ClCompile Include="../../src/xenvif/poller.c" />
<ClCompile Include="../../src/xenvif/receiver.c" />
<ClCompile Include="../../src/xenvif/registry.c" />
<ClCompile Include="../../src/xenvif/settings.c" />
projects / pvdrivers / win / xenvif.git / commitdiff