win-pvdrivers
changeset 535:285d80861ecf
using sg for the tx path - improves performance by about 180% for me
| author | James Harper <james.harper@bendigoit.com.au> |
|---|---|
| date | Tue Jan 27 00:47:02 2009 +1100 (2009-01-27) |
| parents | 1d13cbc9a3b0 |
| children | 1d39de3ab8d6 |
| files | xennet/xennet.c xennet/xennet.h xennet/xennet_common.c xennet/xennet_tx.c |
line diff
1.1 --- a/xennet/xennet.c Tue Jan 27 00:46:33 2009 +1100 1.2 +++ b/xennet/xennet.c Tue Jan 27 00:47:02 2009 +1100 1.3 @@ -334,15 +334,16 @@ XenNet_Init( 1.4 xi->rx_min_target = RX_DFL_MIN_TARGET; 1.5 xi->rx_max_target = RX_MAX_TARGET; 1.6 xi->inactive = TRUE; 1.7 - NdisMSetAttributesEx(xi->adapter_handle, (NDIS_HANDLE) xi, 1.8 - 0, NDIS_ATTRIBUTE_DESERIALIZE|NDIS_ATTRIBUTE_SURPRISE_REMOVE_OK 1.9 + NdisMSetAttributesEx(xi->adapter_handle, (NDIS_HANDLE) xi, 0, 1.10 #ifdef NDIS51_MINIPORT 1.11 - |NDIS_ATTRIBUTE_USES_SAFE_BUFFER_APIS 1.12 + NDIS_ATTRIBUTE_USES_SAFE_BUFFER_APIS 1.13 #endif 1.14 - , 1.15 + |NDIS_ATTRIBUTE_DESERIALIZE 1.16 + |NDIS_ATTRIBUTE_SURPRISE_REMOVE_OK 1.17 + |NDIS_ATTRIBUTE_BUS_MASTER, 1.18 NdisInterfaceInternal); 1.19 xi->multicast_list_size = 0; 1.20 - 1.21 + 1.22 nrl_length = 0; 1.23 NdisMQueryAdapterResources(&status, WrapperConfigurationContext, 1.24 NULL, (PUINT)&nrl_length); 1.25 @@ -409,9 +410,6 @@ XenNet_Init( 1.26 1.27 KeInitializeDpc(&xi->suspend_dpc, XenNet_SuspendResume, xi); 1.28 1.29 - InitializeListHead(&xi->tx_waiting_pkt_list); 1.30 - InitializeListHead(&xi->tx_sent_pkt_list); 1.31 - 1.32 NdisAllocatePacketPool(&status, &xi->packet_pool, XN_RX_QUEUE_LEN * 8, 1.33 PROTOCOL_RESERVED_SIZE_IN_PACKET); 1.34 if (status != NDIS_STATUS_SUCCESS) 1.35 @@ -420,15 +418,6 @@ XenNet_Init( 1.36 status = NDIS_STATUS_RESOURCES; 1.37 goto err; 1.38 } 1.39 - //IS THIS NECESSARY??? NdisSetPacketPoolProtocolId(xi->packet_pool, NDIS_PROTOCOL_ID_TCP_IP); 1.40 - 1.41 - NdisAllocateBufferPool(&status, &xi->buffer_pool, XN_RX_QUEUE_LEN); 1.42 - if (status != NDIS_STATUS_SUCCESS) 1.43 - { 1.44 - KdPrint(("NdisAllocateBufferPool failed with 0x%x\n", status)); 1.45 - status = NDIS_STATUS_RESOURCES; 1.46 - goto err; 1.47 - } 1.48 1.49 NdisMGetDeviceProperty(MiniportAdapterHandle, &xi->pdo, &xi->fdo, 1.50 &xi->lower_do, NULL, NULL); 1.51 @@ -608,7 +597,7 @@ XenNet_Init( 1.52 status = xi->vectors.XenPci_XenConfigDevice(xi->vectors.context); 1.53 if (!NT_SUCCESS(status)) 1.54 { 1.55 - KdPrint(("Failed to complete device configuration\n", status)); 1.56 + KdPrint(("Failed to complete device configuration (%08x)\n", status)); 1.57 goto err; 1.58 } 1.59 1.60 @@ -616,7 +605,14 @@ XenNet_Init( 1.61 1.62 if (!NT_SUCCESS(status)) 1.63 { 1.64 - KdPrint(("Failed to complete device configuration\n", status)); 1.65 + KdPrint(("Failed to complete device configuration (%08x)\n", status)); 1.66 + goto err; 1.67 + } 1.68 + 1.69 + status = NdisMInitializeScatterGatherDma(xi->adapter_handle, TRUE, xi->config_gso); 1.70 + if (!NT_SUCCESS(status)) 1.71 + { 1.72 + KdPrint(("NdisMInitializeScatterGatherDma failed (%08x)\n", status)); 1.73 goto err; 1.74 } 1.75 1.76 @@ -706,7 +702,6 @@ XenNet_Halt( 1.77 XenNet_TxShutdown(xi); 1.78 XenNet_RxShutdown(xi); 1.79 1.80 - NdisFreeBufferPool(xi->buffer_pool); 1.81 NdisFreePacketPool(xi->packet_pool); 1.82 1.83 NdisFreeMemory(xi, 0, 0); // <= DISPATCH_LEVEL
2.1 --- a/xennet/xennet.h Tue Jan 27 00:46:33 2009 +1100 2.2 +++ b/xennet/xennet.h Tue Jan 27 00:47:02 2009 +1100 2.3 @@ -174,11 +174,25 @@ SET_NET_ULONG(PVOID ptr, ULONG data) 2.4 2.5 #define MAX_BUFFERS_PER_PACKET NET_RX_RING_SIZE 2.6 2.7 -typedef struct { 2.8 - ULONG reference_count; 2.9 - PNDIS_BUFFER mdls[MAX_BUFFERS_PER_PACKET]; 2.10 - ULONG mdl_count; 2.11 -} mdl_alloc_t; 2.12 +#define MAX_ETH_HEADER_SIZE 14 2.13 +#define MIN_IP4_HEADER_SIZE 20 2.14 +#define MAX_IP4_HEADER_SIZE (15 * 4) 2.15 +#define MIN_TCP_HEADER_SIZE 20 2.16 +#define MAX_TCP_HEADER_SIZE (15 * 4) 2.17 +#define MAX_PKT_HEADER_SIZE (MAX_ETH_HEADER_SIZE + MAX_IP_HEADER_SIZE + MAX_TCP_HEADER_SIZE) 2.18 + 2.19 +typedef struct 2.20 +{ 2.21 + ULONG id; 2.22 + PHYSICAL_ADDRESS logical; 2.23 + PVOID virtual; 2.24 +} shared_buffer_t; 2.25 + 2.26 +typedef struct 2.27 +{ 2.28 + PNDIS_PACKET packet; /* only set on the last packet */ 2.29 + shared_buffer_t *sb; 2.30 +} tx_shadow_t; 2.31 2.32 typedef struct { 2.33 PNDIS_BUFFER mdls[MAX_BUFFERS_PER_PACKET]; 2.34 @@ -196,7 +210,7 @@ typedef struct { 2.35 ULONG first_buffer_length; 2.36 ULONG header_length; 2.37 UCHAR ip_proto; 2.38 - USHORT total_length; 2.39 + ULONG total_length; 2.40 USHORT ip4_header_length; 2.41 USHORT ip4_length; 2.42 USHORT tcp_header_length; 2.43 @@ -239,7 +253,6 @@ struct xennet_info 2.44 /* NDIS-related vars */ 2.45 NDIS_HANDLE adapter_handle; 2.46 NDIS_HANDLE packet_pool; 2.47 - NDIS_HANDLE buffer_pool; 2.48 NDIS_MINIPORT_INTERRUPT interrupt; 2.49 ULONG packet_filter; 2.50 int connected; 2.51 @@ -261,14 +274,20 @@ struct xennet_info 2.52 /* tx related - protected by tx_lock */ 2.53 KSPIN_LOCK tx_lock; 2.54 LIST_ENTRY tx_waiting_pkt_list; 2.55 - LIST_ENTRY tx_sent_pkt_list; 2.56 struct netif_tx_front_ring tx; 2.57 + ULONG tx_ring_free; 2.58 + tx_shadow_t tx_shadows[NET_TX_RING_SIZE]; 2.59 + NDIS_HANDLE tx_buffer_pool; 2.60 +#define TX_SHARED_BUFFER_SIZE (PAGE_SIZE >> 3) /* 512 */ 2.61 +//#define TX_SHARED_BUFFERS (NET_TX_RING_SIZE >> 4) 2.62 +#define TX_SHARED_BUFFERS (NET_TX_RING_SIZE) 2.63 ULONG tx_id_free; 2.64 - ULONG tx_no_id_used; 2.65 USHORT tx_id_list[NET_TX_RING_SIZE]; 2.66 - //PNDIS_PACKET tx_pkts[NET_TX_RING_SIZE]; 2.67 - PNDIS_BUFFER tx_mdls[NET_TX_RING_SIZE]; 2.68 - freelist_t tx_freelist; 2.69 + ULONG tx_sb_free; 2.70 + ULONG tx_sb_list[TX_SHARED_BUFFERS]; 2.71 + shared_buffer_t tx_sbs[TX_SHARED_BUFFERS]; 2.72 + 2.73 + //freelist_t tx_freelist; 2.74 KDPC tx_dpc; 2.75 2.76 /* rx_related - protected by rx_lock */ 2.77 @@ -319,12 +338,6 @@ struct xennet_info 2.78 2.79 } typedef xennet_info_t; 2.80 2.81 - 2.82 -//NDIS_STATUS 2.83 -//XenNet_RxBufferCheck(struct xennet_info *xi, BOOLEAN is_timer); 2.84 -//VOID 2.85 -//XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2); 2.86 - 2.87 VOID DDKAPI 2.88 XenNet_ReturnPacket( 2.89 IN NDIS_HANDLE MiniportAdapterContext, 2.90 @@ -343,8 +356,6 @@ XenNet_RxResumeStart(xennet_info_t *xi); 2.91 VOID 2.92 XenNet_RxResumeEnd(xennet_info_t *xi); 2.93 2.94 -//NDIS_STATUS 2.95 -//XenNet_TxBufferGC(struct xennet_info *xi); 2.96 VOID 2.97 XenNet_TxBufferGC(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2); 2.98
3.1 --- a/xennet/xennet_common.c Tue Jan 27 00:46:33 2009 +1100 3.2 +++ b/xennet/xennet_common.c Tue Jan 27 00:47:02 2009 +1100 3.3 @@ -34,7 +34,7 @@ XenNet_BuildHeader(packet_info_t *pi, UL 3.4 3.5 if (new_header_size <= pi->header_length) 3.6 { 3.7 - //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size < pi->header_length\n")); 3.8 + //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size <= pi->header_length\n")); 3.9 return TRUE; 3.10 } 3.11 3.12 @@ -48,11 +48,12 @@ XenNet_BuildHeader(packet_info_t *pi, UL 3.13 { 3.14 //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size <= pi->first_buffer_length\n")); 3.15 pi->header_length = new_header_size; 3.16 + pi->curr_mdl_offset = (USHORT)new_header_size; 3.17 return TRUE; 3.18 } 3.19 else if (pi->header != pi->header_data) 3.20 { 3.21 - //KdPrint((__DRIVER_NAME " Using header_data\n")); 3.22 + //KdPrint((__DRIVER_NAME " Switching to header_data\n")); 3.23 memcpy(pi->header_data, pi->header, pi->header_length); 3.24 pi->header = pi->header_data; 3.25 } 3.26 @@ -74,7 +75,7 @@ XenNet_BuildHeader(packet_info_t *pi, UL 3.27 //KdPrint((__DRIVER_NAME " B copy_size = %d\n", copy_size)); 3.28 memcpy(pi->header + pi->header_length, 3.29 (PUCHAR)MmGetMdlVirtualAddress(current_mdl) + pi->curr_mdl_offset, copy_size); 3.30 - pi->curr_mdl_offset += copy_size; 3.31 + pi->curr_mdl_offset = pi->curr_mdl_offset + copy_size; 3.32 pi->header_length += copy_size; 3.33 bytes_remaining -= copy_size; 3.34 } 3.35 @@ -102,7 +103,7 @@ XenNet_ParsePacketHeader(packet_info_t * 3.36 3.37 ASSERT(pi->mdls[0]); 3.38 3.39 - NdisQueryBufferSafe(pi->mdls[0], (PVOID) &pi->header, &pi->first_buffer_length, NormalPagePriority); 3.40 + NdisQueryBufferSafe(pi->mdls[0], (PVOID)&pi->header, &pi->first_buffer_length, NormalPagePriority); 3.41 3.42 pi->header_length = 0; 3.43 pi->curr_mdl_index = 0; 3.44 @@ -151,7 +152,7 @@ XenNet_ParsePacketHeader(packet_info_t * 3.45 } 3.46 break; 3.47 default: 3.48 - KdPrint((__DRIVER_NAME " Not IP (%d)\n", GET_NET_PUSHORT(&pi->header[12]))); 3.49 + //KdPrint((__DRIVER_NAME " Not IP (%d)\n", GET_NET_PUSHORT(&pi->header[12]))); 3.50 return PARSE_UNKNOWN_TYPE; 3.51 } 3.52 pi->ip_proto = pi->header[XN_HDR_SIZE + 9];
4.1 --- a/xennet/xennet_tx.c Tue Jan 27 00:46:33 2009 +1100 4.2 +++ b/xennet/xennet_tx.c Tue Jan 27 00:47:02 2009 +1100 4.3 @@ -20,39 +20,15 @@ Foundation, Inc., 51 Franklin Street, Fi 4.4 4.5 #include "xennet.h" 4.6 4.7 -#define FREELIST_ID_ERROR 0xFFFF 4.8 - 4.9 -static ULONG 4.10 -free_requests(struct xennet_info *xi) 4.11 -{ 4.12 - return xi->tx_id_free - xi->tx_no_id_used; 4.13 -} 4.14 - 4.15 static USHORT 4.16 get_id_from_freelist(struct xennet_info *xi) 4.17 { 4.18 - if (xi->tx_id_free - xi->tx_no_id_used == 0) 4.19 - { 4.20 - KdPrint((__DRIVER_NAME " Out of id's\n")); 4.21 - return FREELIST_ID_ERROR; 4.22 - } 4.23 + ASSERT(xi->tx_id_free); 4.24 xi->tx_id_free--; 4.25 4.26 return xi->tx_id_list[xi->tx_id_free]; 4.27 } 4.28 4.29 -static USHORT 4.30 -get_no_id_from_freelist(struct xennet_info *xi) 4.31 -{ 4.32 - if (xi->tx_id_free - xi->tx_no_id_used == 0) 4.33 - { 4.34 - KdPrint((__DRIVER_NAME " Out of no_id's\n")); 4.35 - return FREELIST_ID_ERROR; 4.36 - } 4.37 - xi->tx_no_id_used++; 4.38 - return 0; 4.39 -} 4.40 - 4.41 static VOID 4.42 put_id_on_freelist(struct xennet_info *xi, USHORT id) 4.43 { 4.44 @@ -60,37 +36,27 @@ put_id_on_freelist(struct xennet_info *x 4.45 xi->tx_id_free++; 4.46 } 4.47 4.48 -static VOID 4.49 -put_no_id_on_freelist(struct xennet_info *xi) 4.50 +static shared_buffer_t * 4.51 +get_sb_from_freelist(struct xennet_info *xi) 4.52 { 4.53 - xi->tx_no_id_used--; 4.54 -} 4.55 -#define SWAP_USHORT(x) (USHORT)((((x & 0xFF) << 8)|((x >> 8) & 0xFF))) 4.56 - 4.57 -/* Place a buffer on tx ring. */ 4.58 -static struct netif_tx_request* 4.59 -XenNet_PutOnTxRing( 4.60 - struct xennet_info *xi, 4.61 - PMDL mdl, 4.62 - uint16_t flags) 4.63 -{ 4.64 - struct netif_tx_request *tx; 4.65 + if (xi->tx_sb_free == 0) 4.66 + { 4.67 + KdPrint((__DRIVER_NAME " Out of sb's\n")); 4.68 + return NULL; 4.69 + } 4.70 + xi->tx_sb_free--; 4.71 4.72 - unsigned short id; 4.73 + return &xi->tx_sbs[xi->tx_sb_list[xi->tx_sb_free]]; 4.74 +} 4.75 4.76 - id = get_id_from_freelist(xi); 4.77 - ASSERT(id != FREELIST_ID_ERROR); 4.78 - //ASSERT(xi->tx_pkts[id] == NULL); 4.79 - tx = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt); 4.80 - tx->gref = get_grant_ref(mdl); 4.81 - xi->tx_mdls[id] = mdl; 4.82 - tx->id = id; 4.83 - tx->offset = 0; 4.84 - tx->size = (USHORT)MmGetMdlByteCount(mdl); 4.85 - tx->flags = flags; 4.86 +static VOID 4.87 +put_sb_on_freelist(struct xennet_info *xi, shared_buffer_t *sb) 4.88 +{ 4.89 + xi->tx_sb_list[xi->tx_sb_free] = sb->id; 4.90 + xi->tx_sb_free++; 4.91 +} 4.92 4.93 - return tx; 4.94 -} 4.95 +#define SWAP_USHORT(x) (USHORT)((((x & 0xFF) << 8)|((x >> 8) & 0xFF))) 4.96 4.97 /* Called at DISPATCH_LEVEL with tx_lock held */ 4.98 /* 4.99 @@ -99,40 +65,42 @@ XenNet_PutOnTxRing( 4.100 static BOOLEAN 4.101 XenNet_HWSendPacket(struct xennet_info *xi, PNDIS_PACKET packet) 4.102 { 4.103 - struct netif_tx_request *tx = NULL; 4.104 + struct netif_tx_request *tx0 = NULL; 4.105 + struct netif_tx_request *txN = NULL; 4.106 struct netif_extra_info *ei = NULL; 4.107 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info; 4.108 //UINT total_packet_length; 4.109 ULONG mss = 0; 4.110 - PMDL in_mdl; 4.111 - PUCHAR in_buffer = NULL; 4.112 - PUCHAR out_buffer; 4.113 - USHORT in_remaining; 4.114 - USHORT out_remaining; 4.115 uint16_t flags = NETTXF_more_data; 4.116 packet_info_t pi; 4.117 BOOLEAN ndis_lso = FALSE; 4.118 BOOLEAN xen_gso = FALSE; 4.119 - int pages_required; 4.120 - int page_num; 4.121 - USHORT copied; 4.122 - UINT first_buffer_length; /* not used */ 4.123 - UINT total_length; 4.124 + //ULONG remaining; 4.125 + PSCATTER_GATHER_LIST sg; 4.126 + ULONG sg_element = 0; 4.127 + ULONG sg_offset = 0; 4.128 + ULONG parse_result; 4.129 + shared_buffer_t *shared_buf = NULL; 4.130 + ULONG i; 4.131 + 4.132 + //FUNCTION_ENTER(); 4.133 4.134 XenNet_ClearPacketInfo(&pi); 4.135 - NdisGetFirstBufferFromPacketSafe(packet, &in_mdl, &pi.header, &first_buffer_length, &total_length, NormalPagePriority); 4.136 - 4.137 - if (!pi.header) 4.138 + NdisQueryPacket(packet, NULL, (PUINT)&pi.mdl_count, &pi.mdls[0], (PUINT)&pi.total_length); 4.139 + //KdPrint((__DRIVER_NAME " A - packet = %p, mdl_count = %d, total_length = %d\n", packet, pi.mdl_count, pi.total_length)); 4.140 + for (i = 1; i < pi.mdl_count; i++) 4.141 { 4.142 - KdPrint((__DRIVER_NAME " NdisGetFirstBufferFromPacketSafe failed\n")); 4.143 - return FALSE; 4.144 + NdisGetNextBuffer(pi.mdls[i - 1], &pi.mdls[i]) 4.145 } 4.146 - 4.147 - if (!total_length) 4.148 +#if 0 4.149 + for (i = 0; i < pi.mdl_count; i++) 4.150 { 4.151 - KdPrint((__DRIVER_NAME " Zero length packet\n")); 4.152 - return TRUE; // we don't want to see this packet again... 4.153 - } 4.154 + KdPrint((__DRIVER_NAME " Aa - mdl[%d] va = %p, length = %d\n", i, MmGetMdlVirtualAddress(pi.mdls[i]), MmGetMdlByteCount(pi.mdls[i]))); 4.155 + } 4.156 +#endif 4.157 + 4.158 + parse_result = XenNet_ParsePacketHeader(&pi); 4.159 + //KdPrint((__DRIVER_NAME " B\n")); 4.160 4.161 if (NDIS_GET_PACKET_PROTOCOL_TYPE(packet) == NDIS_PROTOCOL_ID_TCP_IP) 4.162 { 4.163 @@ -179,7 +147,6 @@ XenNet_HWSendPacket(struct xennet_info * 4.164 //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE); 4.165 //return TRUE; 4.166 } 4.167 - 4.168 } 4.169 4.170 mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo)); 4.171 @@ -194,86 +161,23 @@ XenNet_HWSendPacket(struct xennet_info * 4.172 if (mss > xi->setting_max_offload) 4.173 { 4.174 KdPrint((__DRIVER_NAME " Requested MSS (%d) larger than allowed MSS (%d)\n", mss, xi->setting_max_offload)); 4.175 - NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE); 4.176 + //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE); 4.177 + //FUNCTION_EXIT(); 4.178 return TRUE; 4.179 } 4.180 } 4.181 4.182 - if (!mss && total_length > xi->config_mtu + XN_HDR_SIZE) 4.183 + if (pi.mdl_count + !!ndis_lso > xi->tx_ring_free) 4.184 { 4.185 - KdPrint((__DRIVER_NAME " Packet size (%d) larger than MTU (%d) + header (%d). mss = %d\n", total_length, xi->config_mtu, XN_HDR_SIZE, mss)); 4.186 - NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE); 4.187 - return TRUE; 4.188 - } 4.189 - pages_required = (total_length + PAGE_SIZE - 1) / PAGE_SIZE; 4.190 - 4.191 - if (pages_required + !!ndis_lso > (int)free_requests(xi)) 4.192 - { 4.193 - KdPrint((__DRIVER_NAME " Full on send - required = %d, available = %d\n", pages_required + !!ndis_lso, (int)free_requests(xi))); 4.194 + KdPrint((__DRIVER_NAME " Full on send - required = %d, available = %d\n", pi.mdl_count + !!ndis_lso, xi->tx_ring_free)); 4.195 + //FUNCTION_EXIT(); 4.196 return FALSE; 4.197 } 4.198 4.199 - for (page_num = 0, in_remaining = 0; page_num < pages_required; page_num++) 4.200 - { 4.201 - pi.mdls[page_num] = XenFreelist_GetPage(&xi->tx_freelist); 4.202 - if (!pi.mdls[page_num]) 4.203 - { 4.204 - KdPrint((__DRIVER_NAME " Out of buffers on send (fl->page_outstanding = %d)\n", xi->tx_freelist.page_outstanding)); 4.205 - pages_required = page_num; 4.206 - for (page_num = 0; page_num < pages_required; page_num++) 4.207 - { 4.208 - NdisAdjustBufferLength(pi.mdls[page_num], PAGE_SIZE); 4.209 - XenFreelist_PutPage(&xi->tx_freelist, pi.mdls[page_num]); 4.210 - } 4.211 - return FALSE; 4.212 - } 4.213 - out_buffer = MmGetMdlVirtualAddress(pi.mdls[page_num]); 4.214 - out_remaining = (USHORT)min(PAGE_SIZE, total_length - page_num * PAGE_SIZE); 4.215 - NdisAdjustBufferLength(pi.mdls[page_num], out_remaining); 4.216 - while (out_remaining > 0) 4.217 - { 4.218 - if (!in_remaining) 4.219 - { 4.220 - ASSERT(in_mdl); 4.221 - in_buffer = MmGetSystemAddressForMdlSafe(in_mdl, LowPagePriority); 4.222 - ASSERT(in_buffer != NULL); 4.223 - in_remaining = (USHORT)MmGetMdlByteCount(in_mdl); 4.224 - } 4.225 - copied = min(in_remaining, out_remaining); 4.226 - memcpy(out_buffer, in_buffer, copied); 4.227 - in_remaining = in_remaining - copied; 4.228 - in_buffer += copied; 4.229 - out_remaining = out_remaining - copied; 4.230 - out_buffer += copied; 4.231 - if (!in_remaining) 4.232 - in_mdl = in_mdl->Next; 4.233 - } 4.234 - } 4.235 - /* consume any zero length buffers tacked on the end */ 4.236 - while (in_mdl && MmGetMdlByteCount(in_mdl) == 0) 4.237 - in_mdl = in_mdl->Next; 4.238 - 4.239 - if (in_mdl) 4.240 - { 4.241 - KdPrint((__DRIVER_NAME " Something went wrong... analyzing\n")); 4.242 - NdisGetFirstBufferFromPacketSafe(packet, &in_mdl, &pi.header, &first_buffer_length, &total_length, NormalPagePriority); 4.243 - KdPrint((__DRIVER_NAME " total_length = %d\n", total_length)); 4.244 - while (in_mdl) 4.245 - { 4.246 - KdPrint((__DRIVER_NAME " in_mdl = %p\n", in_mdl)); 4.247 - KdPrint((__DRIVER_NAME " MmGetSystemAddressForMdlSafe(in_mdl) = %p\n", MmGetSystemAddressForMdlSafe(in_mdl, LowPagePriority))); 4.248 - KdPrint((__DRIVER_NAME " MmGetMdlByteCount(in_mdl) = %d\n", MmGetMdlByteCount(in_mdl))); 4.249 - in_mdl = in_mdl->Next; 4.250 - } 4.251 - ASSERT(FALSE); 4.252 - } 4.253 - 4.254 if (ndis_lso) 4.255 - { 4.256 - ULONG parse_result = XenNet_ParsePacketHeader(&pi); 4.257 + { 4.258 if (parse_result == PARSE_OK) 4.259 { 4.260 - XenNet_SumIpHeader(MmGetSystemAddressForMdlSafe(pi.mdls[0], NormalPagePriority), pi.ip4_header_length); 4.261 flags |= NETTXF_csum_blank | NETTXF_data_validated; /* these may be implied but not specified when lso is used*/ 4.262 if (pi.tcp_length >= mss) 4.263 { 4.264 @@ -291,21 +195,74 @@ XenNet_HWSendPacket(struct xennet_info * 4.265 } 4.266 } 4.267 4.268 - /* 4.269 - * See io/netif.h. Must put (A) 1st request, then (B) optional extra_info, then 4.270 - * (C) rest of requests on the ring. Only (A) has csum flags. 4.271 - */ 4.272 + sg = (PSCATTER_GATHER_LIST)NDIS_PER_PACKET_INFO_FROM_PACKET(packet, ScatterGatherListPacketInfo); 4.273 + ASSERT(sg != NULL); 4.274 +#if 0 4.275 + for (i = 0; i < sg->NumberOfElements; i++) 4.276 + { 4.277 + KdPrint((__DRIVER_NAME " Ba - sg->Elements[%d] length = %d\n", i, sg->Elements[i].Length)); 4.278 + } 4.279 +#endif 4.280 +/* 4.281 +* See io/netif.h. Must put (A) 1st request, then (B) optional extra_info, then 4.282 +* (C) rest of requests on the ring. Only (A) has csum flags. 4.283 +*/ 4.284 4.285 + //KdPrint((__DRIVER_NAME " C\n")); 4.286 /* (A) */ 4.287 - tx = XenNet_PutOnTxRing(xi, pi.mdls[0], flags); 4.288 - tx->size = (USHORT)total_length; 4.289 +// if we coalesced the header then we want to put that on first, otherwise we put on the first sg element 4.290 + tx0 = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt); 4.291 + tx0->id = 0xFFFF; 4.292 + if (ndis_lso || (pi.header_length && pi.header_length > sg->Elements[sg_element].Length && pi.header == pi.header_data)) 4.293 + { 4.294 + ULONG remaining = pi.header_length; 4.295 + ASSERT(pi.header_length < TX_SHARED_BUFFER_SIZE); 4.296 + //KdPrint((__DRIVER_NAME " D - header_length = %d\n", pi.header_length)); 4.297 + shared_buf = get_sb_from_freelist(xi); 4.298 + memcpy(shared_buf->virtual, pi.header, pi.header_length); 4.299 + XenNet_SumIpHeader(shared_buf->virtual, pi.ip4_header_length); 4.300 + tx0->gref = (grant_ref_t)(shared_buf->logical.QuadPart >> PAGE_SHIFT); 4.301 + tx0->offset = (USHORT)shared_buf->logical.LowPart & (PAGE_SIZE - 1); 4.302 + tx0->size = (USHORT)pi.header_length; 4.303 + ASSERT(tx0->offset + tx0->size <= PAGE_SIZE); 4.304 + ASSERT(tx0->size); 4.305 + /* TODO: if the next buffer contains only a small amount of data then put it on too */ 4.306 + while (remaining) 4.307 + { 4.308 + //KdPrint((__DRIVER_NAME " D - remaining = %d\n", remaining)); 4.309 + //KdPrint((__DRIVER_NAME " Da - sg_element = %d, sg->Elements[sg_element].Length = %d\n", sg_element, sg->Elements[sg_element].Length)); 4.310 + if (sg->Elements[sg_element].Length <= remaining) 4.311 + { 4.312 + remaining -= sg->Elements[sg_element].Length; 4.313 + sg_element++; 4.314 + } 4.315 + else 4.316 + { 4.317 + sg_offset = remaining; 4.318 + remaining = 0; 4.319 + } 4.320 + } 4.321 + } 4.322 + else 4.323 + { 4.324 + //KdPrint((__DRIVER_NAME " E\n")); 4.325 + //KdPrint((__DRIVER_NAME " Eg - sg_element = %d, sg_offset = %d\n", sg_element, sg_offset)); 4.326 + //KdPrint((__DRIVER_NAME " Eh - address = %p, length = %d\n", 4.327 + // sg->Elements[sg_element].Address.LowPart, sg->Elements[sg_element].Length)); 4.328 + tx0->gref = (grant_ref_t)(sg->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT); 4.329 + tx0->offset = (USHORT)sg->Elements[sg_element].Address.LowPart & (PAGE_SIZE - 1); 4.330 + tx0->size = (USHORT)sg->Elements[sg_element].Length; 4.331 + ASSERT(tx0->size); 4.332 + sg_element++; 4.333 + } 4.334 + tx0->flags = flags; 4.335 + txN = tx0; 4.336 xi->tx.req_prod_pvt++; 4.337 4.338 /* (B) */ 4.339 if (xen_gso) 4.340 { 4.341 ASSERT(flags & NETTXF_extra_info); 4.342 - get_no_id_from_freelist(xi); 4.343 ei = (struct netif_extra_info *)RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt); 4.344 ei->type = XEN_NETIF_EXTRA_TYPE_GSO; 4.345 ei->flags = 0; 4.346 @@ -313,28 +270,45 @@ XenNet_HWSendPacket(struct xennet_info * 4.347 ei->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4; 4.348 ei->u.gso.pad = 0; 4.349 ei->u.gso.features = 0; 4.350 - 4.351 xi->tx.req_prod_pvt++; 4.352 } 4.353 4.354 + //KdPrint((__DRIVER_NAME " F\n")); 4.355 /* (C) */ 4.356 - for (page_num = 1; page_num < pages_required; page_num++) 4.357 + while (sg_element < sg->NumberOfElements) 4.358 { 4.359 - tx = XenNet_PutOnTxRing(xi, pi.mdls[page_num], NETTXF_more_data); 4.360 + //KdPrint((__DRIVER_NAME " G - sg_element = %d, sg_offset = %d\n", sg_element, sg_offset)); 4.361 + //KdPrint((__DRIVER_NAME " H - address = %p, length = %d\n", 4.362 + // sg->Elements[sg_element].Address.LowPart + sg_offset, sg->Elements[sg_element].Length - sg_offset)); 4.363 + txN = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt); 4.364 + txN->id = 0xFFFF; 4.365 + txN->gref = (grant_ref_t)(sg->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT); 4.366 + txN->offset = (USHORT)(sg->Elements[sg_element].Address.LowPart + sg_offset) & (PAGE_SIZE - 1); 4.367 + txN->size = (USHORT)(sg->Elements[sg_element].Length - sg_offset); 4.368 + ASSERT(txN->offset + txN->size <= PAGE_SIZE); 4.369 + ASSERT(txN->size); 4.370 + tx0->size = tx0->size + txN->size; 4.371 + txN->flags = NETTXF_more_data; 4.372 + sg_element++; 4.373 + sg_offset = 0; 4.374 xi->tx.req_prod_pvt++; 4.375 } 4.376 - 4.377 - /* only set the packet on the last buffer, clear more_data */ 4.378 - tx->flags &= ~NETTXF_more_data; 4.379 + txN->flags &= ~NETTXF_more_data; 4.380 + txN->id = get_id_from_freelist(xi); 4.381 +//KdPrint((__DRIVER_NAME " send - id = %d\n", tx0->id)); 4.382 + xi->tx_shadows[txN->id].packet = packet; 4.383 + xi->tx_shadows[txN->id].sb = shared_buf; 4.384 4.385 if (ndis_lso) 4.386 { 4.387 + //KdPrint((__DRIVER_NAME " TcpLargeSendPacketInfo = %d\n", pi.tcp_length)); 4.388 NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo) = UlongToPtr(pi.tcp_length); 4.389 } 4.390 4.391 xi->stat_tx_ok++; 4.392 4.393 - NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS); 4.394 + //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS); 4.395 + //FUNCTION_EXIT(); 4.396 return TRUE; 4.397 } 4.398 4.399 @@ -346,7 +320,6 @@ XenNet_SendQueuedPackets(struct xennet_i 4.400 PLIST_ENTRY entry; 4.401 PNDIS_PACKET packet; 4.402 int notify; 4.403 - BOOLEAN success; 4.404 4.405 //FUNCTION_ENTER(); 4.406 4.407 @@ -356,10 +329,7 @@ XenNet_SendQueuedPackets(struct xennet_i 4.408 { 4.409 packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]); 4.410 //KdPrint((__DRIVER_NAME " Packet ready to send\n")); 4.411 - success = XenNet_HWSendPacket(xi, packet); 4.412 - if (success) 4.413 - InsertTailList(&xi->tx_sent_pkt_list, entry); 4.414 - else 4.415 + if (!XenNet_HWSendPacket(xi, packet)) 4.416 { 4.417 InsertHeadList(&xi->tx_waiting_pkt_list, entry); 4.418 break; 4.419 @@ -375,52 +345,15 @@ XenNet_SendQueuedPackets(struct xennet_i 4.420 //FUNCTION_EXIT(); 4.421 } 4.422 4.423 -// Called at <= DISPATCH_LEVEL with tx spinlock _NOT_ held 4.424 -static VOID 4.425 -XenNet_ReturnSentPackets(struct xennet_info *xi) 4.426 -{ 4.427 - PLIST_ENTRY entry; 4.428 - PNDIS_PACKET packets[32]; 4.429 - int packet_index = 0; 4.430 - int i = 0; 4.431 - KIRQL old_irql; 4.432 - 4.433 - //FUNCTION_ENTER(); 4.434 - 4.435 - old_irql = KeRaiseIrqlToDpcLevel(); 4.436 - KeAcquireSpinLockAtDpcLevel(&xi->tx_lock); 4.437 - entry = RemoveHeadList(&xi->tx_sent_pkt_list); 4.438 - 4.439 - while (entry != &xi->tx_sent_pkt_list) 4.440 - { 4.441 - packets[packet_index++] = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]); 4.442 - entry = RemoveHeadList(&xi->tx_sent_pkt_list); 4.443 - // try to minimize the need to acquire the spinlock repeatedly 4.444 - if (packet_index == 32 || entry == &xi->tx_sent_pkt_list) 4.445 - { 4.446 - KeReleaseSpinLockFromDpcLevel(&xi->tx_lock); 4.447 - for (i = 0; i < packet_index; i++) 4.448 - NdisMSendComplete(xi->adapter_handle, packets[i], NDIS_GET_PACKET_STATUS(packets[i])); 4.449 - if (entry != &xi->tx_sent_pkt_list) /* don't acquire the lock if we have no more packets to SendComplete */ 4.450 - KeAcquireSpinLockAtDpcLevel(&xi->tx_lock); 4.451 - packet_index = 0; 4.452 - } 4.453 - } 4.454 - if (!i) /* i will be == 0 if we didn't SendComplete any packets, and thus we will still have the lock */ 4.455 - KeReleaseSpinLockFromDpcLevel(&xi->tx_lock); 4.456 - KeLowerIrql(old_irql); 4.457 - //FUNCTION_EXIT(); 4.458 -} 4.459 - 4.460 +//ULONG packets_outstanding = 0; 4.461 // Called at DISPATCH_LEVEL 4.462 -//NDIS_STATUS 4.463 -//XenNet_TxBufferGC(struct xennet_info *xi) 4.464 VOID 4.465 XenNet_TxBufferGC(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2) 4.466 { 4.467 struct xennet_info *xi = context; 4.468 RING_IDX cons, prod; 4.469 - unsigned short id; 4.470 + PNDIS_PACKET head = NULL, tail = NULL; 4.471 + PNDIS_PACKET packet; 4.472 4.473 UNREFERENCED_PARAMETER(dpc); 4.474 UNREFERENCED_PARAMETER(arg1); 4.475 @@ -440,23 +373,28 @@ XenNet_TxBufferGC(PKDPC dpc, PVOID conte 4.476 for (cons = xi->tx.rsp_cons; cons != prod; cons++) 4.477 { 4.478 struct netif_tx_response *txrsp; 4.479 - 4.480 txrsp = RING_GET_RESPONSE(&xi->tx, cons); 4.481 - if (txrsp->status == NETIF_RSP_NULL) 4.482 - { 4.483 - put_no_id_on_freelist(xi); 4.484 - continue; // This would be the response to an extra_info packet 4.485 - } 4.486 + if (txrsp->status == NETIF_RSP_NULL || txrsp->id == 0xFFFF) 4.487 + continue; 4.488 4.489 - id = txrsp->id; 4.490 - 4.491 - if (xi->tx_mdls[id]) 4.492 + if (xi->tx_shadows[txrsp->id].sb) 4.493 + { 4.494 + put_sb_on_freelist(xi, xi->tx_shadows[txrsp->id].sb); 4.495 + xi->tx_shadows[txrsp->id].sb = NULL; 4.496 + } 4.497 + 4.498 + if (xi->tx_shadows[txrsp->id].packet) 4.499 { 4.500 - NdisAdjustBufferLength(xi->tx_mdls[id], PAGE_SIZE); 4.501 - XenFreelist_PutPage(&xi->tx_freelist, xi->tx_mdls[id]); 4.502 - xi->tx_mdls[id] = NULL; 4.503 + packet = xi->tx_shadows[txrsp->id].packet; 4.504 + *(PNDIS_PACKET *)&packet->MiniportReservedEx[0] = NULL; 4.505 + if (head) 4.506 + *(PNDIS_PACKET *)&tail->MiniportReservedEx[0] = packet; 4.507 + else 4.508 + head = packet; 4.509 + tail = packet; 4.510 + xi->tx_shadows[txrsp->id].packet = NULL; 4.511 } 4.512 - put_id_on_freelist(xi, id); 4.513 + put_id_on_freelist(xi, txrsp->id); 4.514 } 4.515 4.516 xi->tx.rsp_cons = prod; 4.517 @@ -469,8 +407,14 @@ XenNet_TxBufferGC(PKDPC dpc, PVOID conte 4.518 4.519 KeReleaseSpinLockFromDpcLevel(&xi->tx_lock); 4.520 4.521 - XenNet_ReturnSentPackets(xi); 4.522 - 4.523 + while (head) 4.524 + { 4.525 + packet = (PNDIS_PACKET)head; 4.526 +//KdPrint(("-packet = %p\n", packet)); 4.527 + head = *(PNDIS_PACKET *)&packet->MiniportReservedEx[0]; 4.528 + NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_SUCCESS); 4.529 + } 4.530 +//KdPrint((__DRIVER_NAME " packets_outstanding = %d\n", packets_outstanding)); 4.531 //FUNCTION_EXIT(); 4.532 } 4.533 4.534 @@ -494,8 +438,7 @@ XenNet_SendPackets( 4.535 { 4.536 for (i = 0; i < NumberOfPackets; i++) 4.537 { 4.538 - NDIS_SET_PACKET_STATUS(PacketArray[i], NDIS_STATUS_FAILURE); 4.539 - NdisMSendComplete(xi->adapter_handle, PacketArray[i], NDIS_GET_PACKET_STATUS(PacketArray[i])); 4.540 + NdisMSendComplete(xi->adapter_handle, PacketArray[i], NDIS_STATUS_FAILURE); 4.541 } 4.542 return; 4.543 } 4.544 @@ -506,6 +449,8 @@ XenNet_SendPackets( 4.545 for (i = 0; i < NumberOfPackets; i++) 4.546 { 4.547 packet = PacketArray[i]; 4.548 +//packets_outstanding++; 4.549 +//KdPrint(("+packet = %p\n", packet)); 4.550 ASSERT(packet); 4.551 *(ULONG *)&packet->MiniportReservedEx = 0; 4.552 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)]; 4.553 @@ -517,14 +462,14 @@ XenNet_SendPackets( 4.554 4.555 KeReleaseSpinLock(&xi->tx_lock, OldIrql); 4.556 4.557 - XenNet_ReturnSentPackets(xi); 4.558 - 4.559 //FUNCTION_EXIT(); 4.560 } 4.561 4.562 VOID 4.563 XenNet_TxResumeStart(xennet_info_t *xi) 4.564 { 4.565 + UNREFERENCED_PARAMETER(xi); 4.566 +#if 0 4.567 int i; 4.568 KIRQL old_irql; 4.569 4.570 @@ -538,12 +483,12 @@ XenNet_TxResumeStart(xennet_info_t *xi) 4.571 xi->tx_mdls[i] = NULL; 4.572 } 4.573 } 4.574 - XenFreelist_ResumeStart(&xi->tx_freelist); 4.575 xi->tx_id_free = 0; 4.576 xi->tx_no_id_used = 0; 4.577 for (i = 0; i < NET_TX_RING_SIZE; i++) 4.578 put_id_on_freelist(xi, (USHORT)i); 4.579 KeReleaseSpinLock(&xi->tx_lock, old_irql); 4.580 +#endif 4.581 } 4.582 4.583 VOID 4.584 @@ -552,39 +497,60 @@ XenNet_TxResumeEnd(xennet_info_t *xi) 4.585 KIRQL old_irql; 4.586 4.587 KeAcquireSpinLock(&xi->tx_lock, &old_irql); 4.588 - XenFreelist_ResumeEnd(&xi->tx_freelist); 4.589 XenNet_SendQueuedPackets(xi); 4.590 KeReleaseSpinLock(&xi->tx_lock, old_irql); 4.591 - XenNet_ReturnSentPackets(xi); 4.592 } 4.593 4.594 BOOLEAN 4.595 XenNet_TxInit(xennet_info_t *xi) 4.596 { 4.597 - USHORT i; 4.598 + NDIS_STATUS status; 4.599 + USHORT i, j; 4.600 4.601 KeInitializeSpinLock(&xi->tx_lock); 4.602 KeInitializeDpc(&xi->tx_dpc, XenNet_TxBufferGC, xi); 4.603 /* dpcs are only serialised to a single processor */ 4.604 KeSetTargetProcessorDpc(&xi->tx_dpc, 0); 4.605 //KeSetImportanceDpc(&xi->tx_dpc, HighImportance); 4.606 + InitializeListHead(&xi->tx_waiting_pkt_list); 4.607 + 4.608 + NdisAllocateBufferPool(&status, &xi->tx_buffer_pool, TX_SHARED_BUFFERS); 4.609 + if (status != NDIS_STATUS_SUCCESS) 4.610 + { 4.611 + KdPrint(("NdisAllocateBufferPool failed with 0x%x\n", status)); 4.612 + return FALSE; 4.613 + } 4.614 + 4.615 + xi->tx_ring_free = NET_TX_RING_SIZE; 4.616 + 4.617 + for (i = 0; i < TX_SHARED_BUFFERS / (PAGE_SIZE / TX_SHARED_BUFFER_SIZE); i++) 4.618 + { 4.619 + PVOID virtual; 4.620 + NDIS_PHYSICAL_ADDRESS logical; 4.621 + NdisMAllocateSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, &virtual, &logical); 4.622 + KdPrint((__DRIVER_NAME " Allocated SharedMemory at %p\n", virtual)); 4.623 + for (j = 0; j < PAGE_SIZE / TX_SHARED_BUFFER_SIZE; j++) 4.624 + { 4.625 + ULONG index = i * (PAGE_SIZE / TX_SHARED_BUFFER_SIZE) + j; 4.626 + xi->tx_sbs[index].id = index; 4.627 + xi->tx_sbs[index].virtual = (PUCHAR)virtual + j * TX_SHARED_BUFFER_SIZE; 4.628 + xi->tx_sbs[index].logical.QuadPart = logical.QuadPart + j * TX_SHARED_BUFFER_SIZE; 4.629 + put_sb_on_freelist(xi, &xi->tx_sbs[index]); 4.630 + } 4.631 + } 4.632 4.633 xi->tx_id_free = 0; 4.634 - xi->tx_no_id_used = 0; 4.635 for (i = 0; i < NET_TX_RING_SIZE; i++) 4.636 { 4.637 put_id_on_freelist(xi, i); 4.638 } 4.639 4.640 - XenFreelist_Init(xi, &xi->tx_freelist, &xi->tx_lock); 4.641 - 4.642 return TRUE; 4.643 } 4.644 4.645 /* 4.646 The ring is completely closed down now. We just need to empty anything left 4.647 -on our freelists and harvest anything left on the rings. The freelist timer 4.648 -will still be running though. 4.649 +on our freelists and harvest anything left on the rings. 4.650 */ 4.651 4.652 BOOLEAN 4.653 @@ -592,8 +558,8 @@ XenNet_TxShutdown(xennet_info_t *xi) 4.654 { 4.655 PLIST_ENTRY entry; 4.656 PNDIS_PACKET packet; 4.657 - PMDL mdl; 4.658 - ULONG i; 4.659 + //PMDL mdl; 4.660 + //ULONG i; 4.661 KIRQL OldIrql; 4.662 4.663 FUNCTION_ENTER(); 4.664 @@ -610,7 +576,8 @@ XenNet_TxShutdown(xennet_info_t *xi) 4.665 NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_FAILURE); 4.666 entry = RemoveHeadList(&xi->tx_waiting_pkt_list); 4.667 } 4.668 - 4.669 + 4.670 +#if 0 4.671 /* free sent-but-not-completed packets */ 4.672 for (i = 0; i < NET_TX_RING_SIZE; i++) 4.673 { 4.674 @@ -618,11 +585,11 @@ XenNet_TxShutdown(xennet_info_t *xi) 4.675 if (mdl) 4.676 { 4.677 NdisAdjustBufferLength(xi->tx_mdls[i], PAGE_SIZE); 4.678 - XenFreelist_PutPage(&xi->tx_freelist, xi->tx_mdls[i]); 4.679 + //XenFreelist_PutPage(&xi->tx_freelist, xi->tx_mdls[i]); 4.680 } 4.681 } 4.682 - 4.683 - XenFreelist_Dispose(&xi->tx_freelist); 4.684 +#endif 4.685 + NdisFreeBufferPool(xi->tx_buffer_pool); 4.686 4.687 KeReleaseSpinLock(&xi->tx_lock, OldIrql); 4.688