win-pvdrivers
changeset 535:285d80861ecf
using sg for the tx path - improves performance by about 180% for me
| author | James Harper <james.harper@bendigoit.com.au> |
|---|---|
| date | Tue Jan 27 00:47:02 2009 +1100 (2009-01-27) |
| parents | 1d13cbc9a3b0 |
| children | 1d39de3ab8d6 |
| files | xennet/xennet.c xennet/xennet.h xennet/xennet_common.c xennet/xennet_tx.c |
line diff
1.1 --- a/xennet/xennet.c Tue Jan 27 00:46:33 2009 +1100 1.2 +++ b/xennet/xennet.c Tue Jan 27 00:47:02 2009 +1100 1.3 @@ -334,15 +334,16 @@ XenNet_Init( 1.4 xi->rx_min_target = RX_DFL_MIN_TARGET; 1.5 xi->rx_max_target = RX_MAX_TARGET; 1.6 xi->inactive = TRUE; 1.7 - NdisMSetAttributesEx(xi->adapter_handle, (NDIS_HANDLE) xi, 1.8 - 0, NDIS_ATTRIBUTE_DESERIALIZE|NDIS_ATTRIBUTE_SURPRISE_REMOVE_OK 1.9 + NdisMSetAttributesEx(xi->adapter_handle, (NDIS_HANDLE) xi, 0, 1.10 #ifdef NDIS51_MINIPORT 1.11 - |NDIS_ATTRIBUTE_USES_SAFE_BUFFER_APIS 1.12 + NDIS_ATTRIBUTE_USES_SAFE_BUFFER_APIS 1.13 #endif 1.14 - , 1.15 + |NDIS_ATTRIBUTE_DESERIALIZE 1.16 + |NDIS_ATTRIBUTE_SURPRISE_REMOVE_OK 1.17 + |NDIS_ATTRIBUTE_BUS_MASTER, 1.18 NdisInterfaceInternal); 1.19 xi->multicast_list_size = 0; 1.20 - 1.21 + 1.22 nrl_length = 0; 1.23 NdisMQueryAdapterResources(&status, WrapperConfigurationContext, 1.24 NULL, (PUINT)&nrl_length); 1.25 @@ -409,9 +410,6 @@ XenNet_Init( 1.26 1.27 KeInitializeDpc(&xi->suspend_dpc, XenNet_SuspendResume, xi); 1.28 1.29 - InitializeListHead(&xi->tx_waiting_pkt_list); 1.30 - InitializeListHead(&xi->tx_sent_pkt_list); 1.31 - 1.32 NdisAllocatePacketPool(&status, &xi->packet_pool, XN_RX_QUEUE_LEN * 8, 1.33 PROTOCOL_RESERVED_SIZE_IN_PACKET); 1.34 if (status != NDIS_STATUS_SUCCESS) 1.35 @@ -420,15 +418,6 @@ XenNet_Init( 1.36 status = NDIS_STATUS_RESOURCES; 1.37 goto err; 1.38 } 1.39 - //IS THIS NECESSARY??? NdisSetPacketPoolProtocolId(xi->packet_pool, NDIS_PROTOCOL_ID_TCP_IP); 1.40 - 1.41 - NdisAllocateBufferPool(&status, &xi->buffer_pool, XN_RX_QUEUE_LEN); 1.42 - if (status != NDIS_STATUS_SUCCESS) 1.43 - { 1.44 - KdPrint(("NdisAllocateBufferPool failed with 0x%x\n", status)); 1.45 - status = NDIS_STATUS_RESOURCES; 1.46 - goto err; 1.47 - } 1.48 1.49 NdisMGetDeviceProperty(MiniportAdapterHandle, &xi->pdo, &xi->fdo, 1.50 &xi->lower_do, NULL, NULL); 1.51 @@ -608,7 +597,7 @@ XenNet_Init( 1.52 status = xi->vectors.XenPci_XenConfigDevice(xi->vectors.context); 1.53 if (!NT_SUCCESS(status)) 1.54 { 1.55 - KdPrint(("Failed to complete device configuration\n", status)); 1.56 + KdPrint(("Failed to complete device configuration (%08x)\n", status)); 1.57 goto err; 1.58 } 1.59 1.60 @@ -616,7 +605,14 @@ XenNet_Init( 1.61 1.62 if (!NT_SUCCESS(status)) 1.63 { 1.64 - KdPrint(("Failed to complete device configuration\n", status)); 1.65 + KdPrint(("Failed to complete device configuration (%08x)\n", status)); 1.66 + goto err; 1.67 + } 1.68 + 1.69 + status = NdisMInitializeScatterGatherDma(xi->adapter_handle, TRUE, xi->config_gso); 1.70 + if (!NT_SUCCESS(status)) 1.71 + { 1.72 + KdPrint(("NdisMInitializeScatterGatherDma failed (%08x)\n", status)); 1.73 goto err; 1.74 } 1.75 1.76 @@ -706,7 +702,6 @@ XenNet_Halt( 1.77 XenNet_TxShutdown(xi); 1.78 XenNet_RxShutdown(xi); 1.79 1.80 - NdisFreeBufferPool(xi->buffer_pool); 1.81 NdisFreePacketPool(xi->packet_pool); 1.82 1.83 NdisFreeMemory(xi, 0, 0); // <= DISPATCH_LEVEL
2.1 --- a/xennet/xennet.h Tue Jan 27 00:46:33 2009 +1100 2.2 +++ b/xennet/xennet.h Tue Jan 27 00:47:02 2009 +1100 2.3 @@ -174,11 +174,25 @@ SET_NET_ULONG(PVOID ptr, ULONG data) 2.4 2.5 #define MAX_BUFFERS_PER_PACKET NET_RX_RING_SIZE 2.6 2.7 -typedef struct { 2.8 - ULONG reference_count; 2.9 - PNDIS_BUFFER mdls[MAX_BUFFERS_PER_PACKET]; 2.10 - ULONG mdl_count; 2.11 -} mdl_alloc_t; 2.12 +#define MAX_ETH_HEADER_SIZE 14 2.13 +#define MIN_IP4_HEADER_SIZE 20 2.14 +#define MAX_IP4_HEADER_SIZE (15 * 4) 2.15 +#define MIN_TCP_HEADER_SIZE 20 2.16 +#define MAX_TCP_HEADER_SIZE (15 * 4) 2.17 +#define MAX_PKT_HEADER_SIZE (MAX_ETH_HEADER_SIZE + MAX_IP_HEADER_SIZE + MAX_TCP_HEADER_SIZE) 2.18 + 2.19 +typedef struct 2.20 +{ 2.21 + ULONG id; 2.22 + PHYSICAL_ADDRESS logical; 2.23 + PVOID virtual; 2.24 +} shared_buffer_t; 2.25 + 2.26 +typedef struct 2.27 +{ 2.28 + PNDIS_PACKET packet; /* only set on the last packet */ 2.29 + shared_buffer_t *sb; 2.30 +} tx_shadow_t; 2.31 2.32 typedef struct { 2.33 PNDIS_BUFFER mdls[MAX_BUFFERS_PER_PACKET]; 2.34 @@ -196,7 +210,7 @@ typedef struct { 2.35 ULONG first_buffer_length; 2.36 ULONG header_length; 2.37 UCHAR ip_proto; 2.38 - USHORT total_length; 2.39 + ULONG total_length; 2.40 USHORT ip4_header_length; 2.41 USHORT ip4_length; 2.42 USHORT tcp_header_length; 2.43 @@ -239,7 +253,6 @@ struct xennet_info 2.44 /* NDIS-related vars */ 2.45 NDIS_HANDLE adapter_handle; 2.46 NDIS_HANDLE packet_pool; 2.47 - NDIS_HANDLE buffer_pool; 2.48 NDIS_MINIPORT_INTERRUPT interrupt; 2.49 ULONG packet_filter; 2.50 int connected; 2.51 @@ -261,14 +274,20 @@ struct xennet_info 2.52 /* tx related - protected by tx_lock */ 2.53 KSPIN_LOCK tx_lock; 2.54 LIST_ENTRY tx_waiting_pkt_list; 2.55 - LIST_ENTRY tx_sent_pkt_list; 2.56 struct netif_tx_front_ring tx; 2.57 + ULONG tx_ring_free; 2.58 + tx_shadow_t tx_shadows[NET_TX_RING_SIZE]; 2.59 + NDIS_HANDLE tx_buffer_pool; 2.60 +#define TX_SHARED_BUFFER_SIZE (PAGE_SIZE >> 3) /* 512 */ 2.61 +//#define TX_SHARED_BUFFERS (NET_TX_RING_SIZE >> 4) 2.62 +#define TX_SHARED_BUFFERS (NET_TX_RING_SIZE) 2.63 ULONG tx_id_free; 2.64 - ULONG tx_no_id_used; 2.65 USHORT tx_id_list[NET_TX_RING_SIZE]; 2.66 - //PNDIS_PACKET tx_pkts[NET_TX_RING_SIZE]; 2.67 - PNDIS_BUFFER tx_mdls[NET_TX_RING_SIZE]; 2.68 - freelist_t tx_freelist; 2.69 + ULONG tx_sb_free; 2.70 + ULONG tx_sb_list[TX_SHARED_BUFFERS]; 2.71 + shared_buffer_t tx_sbs[TX_SHARED_BUFFERS]; 2.72 + 2.73 + //freelist_t tx_freelist; 2.74 KDPC tx_dpc; 2.75 2.76 /* rx_related - protected by rx_lock */ 2.77 @@ -319,12 +338,6 @@ struct xennet_info 2.78 2.79 } typedef xennet_info_t; 2.80 2.81 - 2.82 -//NDIS_STATUS 2.83 -//XenNet_RxBufferCheck(struct xennet_info *xi, BOOLEAN is_timer); 2.84 -//VOID 2.85 -//XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2); 2.86 - 2.87 VOID DDKAPI 2.88 XenNet_ReturnPacket( 2.89 IN NDIS_HANDLE MiniportAdapterContext, 2.90 @@ -343,8 +356,6 @@ XenNet_RxResumeStart(xennet_info_t *xi); 2.91 VOID 2.92 XenNet_RxResumeEnd(xennet_info_t *xi); 2.93 2.94 -//NDIS_STATUS 2.95 -//XenNet_TxBufferGC(struct xennet_info *xi); 2.96 VOID 2.97 XenNet_TxBufferGC(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2); 2.98
3.1 --- a/xennet/xennet_common.c Tue Jan 27 00:46:33 2009 +1100 3.2 +++ b/xennet/xennet_common.c Tue Jan 27 00:47:02 2009 +1100 3.3 @@ -34,7 +34,7 @@ XenNet_BuildHeader(packet_info_t *pi, UL 3.4 3.5 if (new_header_size <= pi->header_length) 3.6 { 3.7 - //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size < pi->header_length\n")); 3.8 + //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size <= pi->header_length\n")); 3.9 return TRUE; 3.10 } 3.11 3.12 @@ -48,11 +48,12 @@ XenNet_BuildHeader(packet_info_t *pi, UL 3.13 { 3.14 //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size <= pi->first_buffer_length\n")); 3.15 pi->header_length = new_header_size; 3.16 + pi->curr_mdl_offset = (USHORT)new_header_size; 3.17 return TRUE; 3.18 } 3.19 else if (pi->header != pi->header_data) 3.20 { 3.21 - //KdPrint((__DRIVER_NAME " Using header_data\n")); 3.22 + //KdPrint((__DRIVER_NAME " Switching to header_data\n")); 3.23 memcpy(pi->header_data, pi->header, pi->header_length); 3.24 pi->header = pi->header_data; 3.25 } 3.26 @@ -74,7 +75,7 @@ XenNet_BuildHeader(packet_info_t *pi, UL 3.27 //KdPrint((__DRIVER_NAME " B copy_size = %d\n", copy_size)); 3.28 memcpy(pi->header + pi->header_length, 3.29 (PUCHAR)MmGetMdlVirtualAddress(current_mdl) + pi->curr_mdl_offset, copy_size); 3.30 - pi->curr_mdl_offset += copy_size; 3.31 + pi->curr_mdl_offset = pi->curr_mdl_offset + copy_size; 3.32 pi->header_length += copy_size; 3.33 bytes_remaining -= copy_size; 3.34 } 3.35 @@ -102,7 +103,7 @@ XenNet_ParsePacketHeader(packet_info_t * 3.36 3.37 ASSERT(pi->mdls[0]); 3.38 3.39 - NdisQueryBufferSafe(pi->mdls[0], (PVOID) &pi->header, &pi->first_buffer_length, NormalPagePriority); 3.40 + NdisQueryBufferSafe(pi->mdls[0], (PVOID)&pi->header, &pi->first_buffer_length, NormalPagePriority); 3.41 3.42 pi->header_length = 0; 3.43 pi->curr_mdl_index = 0; 3.44 @@ -151,7 +152,7 @@ XenNet_ParsePacketHeader(packet_info_t * 3.45 } 3.46 break; 3.47 default: 3.48 - KdPrint((__DRIVER_NAME " Not IP (%d)\n", GET_NET_PUSHORT(&pi->header[12]))); 3.49 + //KdPrint((__DRIVER_NAME " Not IP (%d)\n", GET_NET_PUSHORT(&pi->header[12]))); 3.50 return PARSE_UNKNOWN_TYPE; 3.51 } 3.52 pi->ip_proto = pi->header[XN_HDR_SIZE + 9];
4.1 --- a/xennet/xennet_tx.c Tue Jan 27 00:46:33 2009 +1100 4.2 +++ b/xennet/xennet_tx.c Tue Jan 27 00:47:02 2009 +1100 4.3 @@ -20,39 +20,15 @@ Foundation, Inc., 51 Franklin Street, Fi 4.4 4.5 #include "xennet.h" 4.6 4.7 -#define FREELIST_ID_ERROR 0xFFFF 4.8 - 4.9 -static ULONG 4.10 -free_requests(struct xennet_info *xi) 4.11 -{ 4.12 - return xi->tx_id_free - xi->tx_no_id_used; 4.13 -} 4.14 - 4.15 static USHORT 4.16 get_id_from_freelist(struct xennet_info *xi) 4.17 { 4.18 - if (xi->tx_id_free - xi->tx_no_id_used == 0) 4.19 - { 4.20 - KdPrint((__DRIVER_NAME " Out of id's\n")); 4.21 - return FREELIST_ID_ERROR; 4.22 - } 4.23 + ASSERT(xi->tx_id_free); 4.24 xi->tx_id_free--; 4.25 4.26 return xi->tx_id_list[xi->tx_id_free]; 4.27 } 4.28 4.29 -static USHORT 4.30 -get_no_id_from_freelist(struct xennet_info *xi) 4.31 -{ 4.32 - if (xi->tx_id_free - xi->tx_no_id_used == 0) 4.33 - { 4.34 - KdPrint((__DRIVER_NAME " Out of no_id's\n")); 4.35 - return FREELIST_ID_ERROR; 4.36 - } 4.37 - xi->tx_no_id_used++; 4.38 - return 0; 4.39 -} 4.40 - 4.41 static VOID 4.42 put_id_on_freelist(struct xennet_info *xi, USHORT id) 4.43 { 4.44 @@ -60,38 +36,28 @@ put_id_on_freelist(struct xennet_info *x 4.45 xi->tx_id_free++; 4.46 } 4.47 4.48 -static VOID 4.49 -put_no_id_on_freelist(struct xennet_info *xi) 4.50 -{ 4.51 - xi->tx_no_id_used--; 4.52 -} 4.53 -#define SWAP_USHORT(x) (USHORT)((((x & 0xFF) << 8)|((x >> 8) & 0xFF))) 4.54 - 4.55 -/* Place a buffer on tx ring. */ 4.56 -static struct netif_tx_request* 4.57 -XenNet_PutOnTxRing( 4.58 - struct xennet_info *xi, 4.59 - PMDL mdl, 4.60 - uint16_t flags) 4.61 +static shared_buffer_t * 4.62 +get_sb_from_freelist(struct xennet_info *xi) 4.63 { 4.64 - struct netif_tx_request *tx; 4.65 - 4.66 - unsigned short id; 4.67 + if (xi->tx_sb_free == 0) 4.68 + { 4.69 + KdPrint((__DRIVER_NAME " Out of sb's\n")); 4.70 + return NULL; 4.71 + } 4.72 + xi->tx_sb_free--; 4.73 4.74 - id = get_id_from_freelist(xi); 4.75 - ASSERT(id != FREELIST_ID_ERROR); 4.76 - //ASSERT(xi->tx_pkts[id] == NULL); 4.77 - tx = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt); 4.78 - tx->gref = get_grant_ref(mdl); 4.79 - xi->tx_mdls[id] = mdl; 4.80 - tx->id = id; 4.81 - tx->offset = 0; 4.82 - tx->size = (USHORT)MmGetMdlByteCount(mdl); 4.83 - tx->flags = flags; 4.84 + return &xi->tx_sbs[xi->tx_sb_list[xi->tx_sb_free]]; 4.85 +} 4.86 4.87 - return tx; 4.88 +static VOID 4.89 +put_sb_on_freelist(struct xennet_info *xi, shared_buffer_t *sb) 4.90 +{ 4.91 + xi->tx_sb_list[xi->tx_sb_free] = sb->id; 4.92 + xi->tx_sb_free++; 4.93 } 4.94 4.95 +#define SWAP_USHORT(x) (USHORT)((((x & 0xFF) << 8)|((x >> 8) & 0xFF))) 4.96 + 4.97 /* Called at DISPATCH_LEVEL with tx_lock held */ 4.98 /* 4.99 * Send one NDIS_PACKET. This may involve multiple entries on TX ring. 4.100 @@ -99,40 +65,42 @@ XenNet_PutOnTxRing( 4.101 static BOOLEAN 4.102 XenNet_HWSendPacket(struct xennet_info *xi, PNDIS_PACKET packet) 4.103 { 4.104 - struct netif_tx_request *tx = NULL; 4.105 + struct netif_tx_request *tx0 = NULL; 4.106 + struct netif_tx_request *txN = NULL; 4.107 struct netif_extra_info *ei = NULL; 4.108 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info; 4.109 //UINT total_packet_length; 4.110 ULONG mss = 0; 4.111 - PMDL in_mdl; 4.112 - PUCHAR in_buffer = NULL; 4.113 - PUCHAR out_buffer; 4.114 - USHORT in_remaining; 4.115 - USHORT out_remaining; 4.116 uint16_t flags = NETTXF_more_data; 4.117 packet_info_t pi; 4.118 BOOLEAN ndis_lso = FALSE; 4.119 BOOLEAN xen_gso = FALSE; 4.120 - int pages_required; 4.121 - int page_num; 4.122 - USHORT copied; 4.123 - UINT first_buffer_length; /* not used */ 4.124 - UINT total_length; 4.125 + //ULONG remaining; 4.126 + PSCATTER_GATHER_LIST sg; 4.127 + ULONG sg_element = 0; 4.128 + ULONG sg_offset = 0; 4.129 + ULONG parse_result; 4.130 + shared_buffer_t *shared_buf = NULL; 4.131 + ULONG i; 4.132 + 4.133 + //FUNCTION_ENTER(); 4.134 4.135 XenNet_ClearPacketInfo(&pi); 4.136 - NdisGetFirstBufferFromPacketSafe(packet, &in_mdl, &pi.header, &first_buffer_length, &total_length, NormalPagePriority); 4.137 - 4.138 - if (!pi.header) 4.139 + NdisQueryPacket(packet, NULL, (PUINT)&pi.mdl_count, &pi.mdls[0], (PUINT)&pi.total_length); 4.140 + //KdPrint((__DRIVER_NAME " A - packet = %p, mdl_count = %d, total_length = %d\n", packet, pi.mdl_count, pi.total_length)); 4.141 + for (i = 1; i < pi.mdl_count; i++) 4.142 { 4.143 - KdPrint((__DRIVER_NAME " NdisGetFirstBufferFromPacketSafe failed\n")); 4.144 - return FALSE; 4.145 + NdisGetNextBuffer(pi.mdls[i - 1], &pi.mdls[i]) 4.146 } 4.147 +#if 0 4.148 + for (i = 0; i < pi.mdl_count; i++) 4.149 + { 4.150 + KdPrint((__DRIVER_NAME " Aa - mdl[%d] va = %p, length = %d\n", i, MmGetMdlVirtualAddress(pi.mdls[i]), MmGetMdlByteCount(pi.mdls[i]))); 4.151 + } 4.152 +#endif 4.153 4.154 - if (!total_length) 4.155 - { 4.156 - KdPrint((__DRIVER_NAME " Zero length packet\n")); 4.157 - return TRUE; // we don't want to see this packet again... 4.158 - } 4.159 + parse_result = XenNet_ParsePacketHeader(&pi); 4.160 + //KdPrint((__DRIVER_NAME " B\n")); 4.161 4.162 if (NDIS_GET_PACKET_PROTOCOL_TYPE(packet) == NDIS_PROTOCOL_ID_TCP_IP) 4.163 { 4.164 @@ -179,7 +147,6 @@ XenNet_HWSendPacket(struct xennet_info * 4.165 //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE); 4.166 //return TRUE; 4.167 } 4.168 - 4.169 } 4.170 4.171 mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo)); 4.172 @@ -194,86 +161,23 @@ XenNet_HWSendPacket(struct xennet_info * 4.173 if (mss > xi->setting_max_offload) 4.174 { 4.175 KdPrint((__DRIVER_NAME " Requested MSS (%d) larger than allowed MSS (%d)\n", mss, xi->setting_max_offload)); 4.176 - NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE); 4.177 + //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE); 4.178 + //FUNCTION_EXIT(); 4.179 return TRUE; 4.180 } 4.181 } 4.182 4.183 - if (!mss && total_length > xi->config_mtu + XN_HDR_SIZE) 4.184 + if (pi.mdl_count + !!ndis_lso > xi->tx_ring_free) 4.185 { 4.186 - KdPrint((__DRIVER_NAME " Packet size (%d) larger than MTU (%d) + header (%d). mss = %d\n", total_length, xi->config_mtu, XN_HDR_SIZE, mss)); 4.187 - NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE); 4.188 - return TRUE; 4.189 - } 4.190 - pages_required = (total_length + PAGE_SIZE - 1) / PAGE_SIZE; 4.191 - 4.192 - if (pages_required + !!ndis_lso > (int)free_requests(xi)) 4.193 - { 4.194 - KdPrint((__DRIVER_NAME " Full on send - required = %d, available = %d\n", pages_required + !!ndis_lso, (int)free_requests(xi))); 4.195 + KdPrint((__DRIVER_NAME " Full on send - required = %d, available = %d\n", pi.mdl_count + !!ndis_lso, xi->tx_ring_free)); 4.196 + //FUNCTION_EXIT(); 4.197 return FALSE; 4.198 } 4.199 4.200 - for (page_num = 0, in_remaining = 0; page_num < pages_required; page_num++) 4.201 - { 4.202 - pi.mdls[page_num] = XenFreelist_GetPage(&xi->tx_freelist); 4.203 - if (!pi.mdls[page_num]) 4.204 - { 4.205 - KdPrint((__DRIVER_NAME " Out of buffers on send (fl->page_outstanding = %d)\n", xi->tx_freelist.page_outstanding)); 4.206 - pages_required = page_num; 4.207 - for (page_num = 0; page_num < pages_required; page_num++) 4.208 - { 4.209 - NdisAdjustBufferLength(pi.mdls[page_num], PAGE_SIZE); 4.210 - XenFreelist_PutPage(&xi->tx_freelist, pi.mdls[page_num]); 4.211 - } 4.212 - return FALSE; 4.213 - } 4.214 - out_buffer = MmGetMdlVirtualAddress(pi.mdls[page_num]); 4.215 - out_remaining = (USHORT)min(PAGE_SIZE, total_length - page_num * PAGE_SIZE); 4.216 - NdisAdjustBufferLength(pi.mdls[page_num], out_remaining); 4.217 - while (out_remaining > 0) 4.218 - { 4.219 - if (!in_remaining) 4.220 - { 4.221 - ASSERT(in_mdl); 4.222 - in_buffer = MmGetSystemAddressForMdlSafe(in_mdl, LowPagePriority); 4.223 - ASSERT(in_buffer != NULL); 4.224 - in_remaining = (USHORT)MmGetMdlByteCount(in_mdl); 4.225 - } 4.226 - copied = min(in_remaining, out_remaining); 4.227 - memcpy(out_buffer, in_buffer, copied); 4.228 - in_remaining = in_remaining - copied; 4.229 - in_buffer += copied; 4.230 - out_remaining = out_remaining - copied; 4.231 - out_buffer += copied; 4.232 - if (!in_remaining) 4.233 - in_mdl = in_mdl->Next; 4.234 - } 4.235 - } 4.236 - /* consume any zero length buffers tacked on the end */ 4.237 - while (in_mdl && MmGetMdlByteCount(in_mdl) == 0) 4.238 - in_mdl = in_mdl->Next; 4.239 - 4.240 - if (in_mdl) 4.241 - { 4.242 - KdPrint((__DRIVER_NAME " Something went wrong... analyzing\n")); 4.243 - NdisGetFirstBufferFromPacketSafe(packet, &in_mdl, &pi.header, &first_buffer_length, &total_length, NormalPagePriority); 4.244 - KdPrint((__DRIVER_NAME " total_length = %d\n", total_length)); 4.245 - while (in_mdl) 4.246 - { 4.247 - KdPrint((__DRIVER_NAME " in_mdl = %p\n", in_mdl)); 4.248 - KdPrint((__DRIVER_NAME " MmGetSystemAddressForMdlSafe(in_mdl) = %p\n", MmGetSystemAddressForMdlSafe(in_mdl, LowPagePriority))); 4.249 - KdPrint((__DRIVER_NAME " MmGetMdlByteCount(in_mdl) = %d\n", MmGetMdlByteCount(in_mdl))); 4.250 - in_mdl = in_mdl->Next; 4.251 - } 4.252 - ASSERT(FALSE); 4.253 - } 4.254 - 4.255 if (ndis_lso) 4.256 - { 4.257 - ULONG parse_result = XenNet_ParsePacketHeader(&pi); 4.258 + { 4.259 if (parse_result == PARSE_OK) 4.260 { 4.261 - XenNet_SumIpHeader(MmGetSystemAddressForMdlSafe(pi.mdls[0], NormalPagePriority), pi.ip4_header_length); 4.262 flags |= NETTXF_csum_blank | NETTXF_data_validated; /* these may be implied but not specified when lso is used*/ 4.263 if (pi.tcp_length >= mss) 4.264 { 4.265 @@ -291,21 +195,74 @@ XenNet_HWSendPacket(struct xennet_info * 4.266 } 4.267 } 4.268 4.269 - /* 4.270 - * See io/netif.h. Must put (A) 1st request, then (B) optional extra_info, then 4.271 - * (C) rest of requests on the ring. Only (A) has csum flags. 4.272 - */ 4.273 + sg = (PSCATTER_GATHER_LIST)NDIS_PER_PACKET_INFO_FROM_PACKET(packet, ScatterGatherListPacketInfo); 4.274 + ASSERT(sg != NULL); 4.275 +#if 0 4.276 + for (i = 0; i < sg->NumberOfElements; i++) 4.277 + { 4.278 + KdPrint((__DRIVER_NAME " Ba - sg->Elements[%d] length = %d\n", i, sg->Elements[i].Length)); 4.279 + } 4.280 +#endif 4.281 +/* 4.282 +* See io/netif.h. Must put (A) 1st request, then (B) optional extra_info, then 4.283 +* (C) rest of requests on the ring. Only (A) has csum flags. 4.284 +*/ 4.285 4.286 + //KdPrint((__DRIVER_NAME " C\n")); 4.287 /* (A) */ 4.288 - tx = XenNet_PutOnTxRing(xi, pi.mdls[0], flags); 4.289 - tx->size = (USHORT)total_length; 4.290 +// if we coalesced the header then we want to put that on first, otherwise we put on the first sg element 4.291 + tx0 = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt); 4.292 + tx0->id = 0xFFFF; 4.293 + if (ndis_lso || (pi.header_length && pi.header_length > sg->Elements[sg_element].Length && pi.header == pi.header_data)) 4.294 + { 4.295 + ULONG remaining = pi.header_length; 4.296 + ASSERT(pi.header_length < TX_SHARED_BUFFER_SIZE); 4.297 + //KdPrint((__DRIVER_NAME " D - header_length = %d\n", pi.header_length)); 4.298 + shared_buf = get_sb_from_freelist(xi); 4.299 + memcpy(shared_buf->virtual, pi.header, pi.header_length); 4.300 + XenNet_SumIpHeader(shared_buf->virtual, pi.ip4_header_length); 4.301 + tx0->gref = (grant_ref_t)(shared_buf->logical.QuadPart >> PAGE_SHIFT); 4.302 + tx0->offset = (USHORT)shared_buf->logical.LowPart & (PAGE_SIZE - 1); 4.303 + tx0->size = (USHORT)pi.header_length; 4.304 + ASSERT(tx0->offset + tx0->size <= PAGE_SIZE); 4.305 + ASSERT(tx0->size); 4.306 + /* TODO: if the next buffer contains only a small amount of data then put it on too */ 4.307 + while (remaining) 4.308 + { 4.309 + //KdPrint((__DRIVER_NAME " D - remaining = %d\n", remaining)); 4.310 + //KdPrint((__DRIVER_NAME " Da - sg_element = %d, sg->Elements[sg_element].Length = %d\n", sg_element, sg->Elements[sg_element].Length)); 4.311 + if (sg->Elements[sg_element].Length <= remaining) 4.312 + { 4.313 + remaining -= sg->Elements[sg_element].Length; 4.314 + sg_element++; 4.315 + } 4.316 + else 4.317 + { 4.318 + sg_offset = remaining; 4.319 + remaining = 0; 4.320 + } 4.321 + } 4.322 + } 4.323 + else 4.324 + { 4.325 + //KdPrint((__DRIVER_NAME " E\n")); 4.326 + //KdPrint((__DRIVER_NAME " Eg - sg_element = %d, sg_offset = %d\n", sg_element, sg_offset)); 4.327 + //KdPrint((__DRIVER_NAME " Eh - address = %p, length = %d\n", 4.328 + // sg->Elements[sg_element].Address.LowPart, sg->Elements[sg_element].Length)); 4.329 + tx0->gref = (grant_ref_t)(sg->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT); 4.330 + tx0->offset = (USHORT)sg->Elements[sg_element].Address.LowPart & (PAGE_SIZE - 1); 4.331 + tx0->size = (USHORT)sg->Elements[sg_element].Length; 4.332 + ASSERT(tx0->size); 4.333 + sg_element++; 4.334 + } 4.335 + tx0->flags = flags; 4.336 + txN = tx0; 4.337 xi->tx.req_prod_pvt++; 4.338 4.339 /* (B) */ 4.340 if (xen_gso) 4.341 { 4.342 ASSERT(flags & NETTXF_extra_info); 4.343 - get_no_id_from_freelist(xi); 4.344 ei = (struct netif_extra_info *)RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt); 4.345 ei->type = XEN_NETIF_EXTRA_TYPE_GSO; 4.346 ei->flags = 0; 4.347 @@ -313,28 +270,45 @@ XenNet_HWSendPacket(struct xennet_info * 4.348 ei->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4; 4.349 ei->u.gso.pad = 0; 4.350 ei->u.gso.features = 0; 4.351 - 4.352 xi->tx.req_prod_pvt++; 4.353 } 4.354 4.355 + //KdPrint((__DRIVER_NAME " F\n")); 4.356 /* (C) */ 4.357 - for (page_num = 1; page_num < pages_required; page_num++) 4.358 + while (sg_element < sg->NumberOfElements) 4.359 { 4.360 - tx = XenNet_PutOnTxRing(xi, pi.mdls[page_num], NETTXF_more_data); 4.361 + //KdPrint((__DRIVER_NAME " G - sg_element = %d, sg_offset = %d\n", sg_element, sg_offset)); 4.362 + //KdPrint((__DRIVER_NAME " H - address = %p, length = %d\n", 4.363 + // sg->Elements[sg_element].Address.LowPart + sg_offset, sg->Elements[sg_element].Length - sg_offset)); 4.364 + txN = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt); 4.365 + txN->id = 0xFFFF; 4.366 + txN->gref = (grant_ref_t)(sg->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT); 4.367 + txN->offset = (USHORT)(sg->Elements[sg_element].Address.LowPart + sg_offset) & (PAGE_SIZE - 1); 4.368 + txN->size = (USHORT)(sg->Elements[sg_element].Length - sg_offset); 4.369 + ASSERT(txN->offset + txN->size <= PAGE_SIZE); 4.370 + ASSERT(txN->size); 4.371 + tx0->size = tx0->size + txN->size; 4.372 + txN->flags = NETTXF_more_data; 4.373 + sg_element++; 4.374 + sg_offset = 0; 4.375 xi->tx.req_prod_pvt++; 4.376 } 4.377 - 4.378 - /* only set the packet on the last buffer, clear more_data */ 4.379 - tx->flags &= ~NETTXF_more_data; 4.380 + txN->flags &= ~NETTXF_more_data; 4.381 + txN->id = get_id_from_freelist(xi); 4.382 +//KdPrint((__DRIVER_NAME " send - id = %d\n", tx0->id)); 4.383 + xi->tx_shadows[txN->id].packet = packet; 4.384 + xi->tx_shadows[txN->id].sb = shared_buf; 4.385 4.386 if (ndis_lso) 4.387 { 4.388 + //KdPrint((__DRIVER_NAME " TcpLargeSendPacketInfo = %d\n", pi.tcp_length)); 4.389 NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo) = UlongToPtr(pi.tcp_length); 4.390 } 4.391 4.392 xi->stat_tx_ok++; 4.393 4.394 - NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS); 4.395 + //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS); 4.396 + //FUNCTION_EXIT(); 4.397 return TRUE; 4.398 } 4.399 4.400 @@ -346,7 +320,6 @@ XenNet_SendQueuedPackets(struct xennet_i 4.401 PLIST_ENTRY entry; 4.402 PNDIS_PACKET packet; 4.403 int notify; 4.404 - BOOLEAN success; 4.405 4.406 //FUNCTION_ENTER(); 4.407 4.408 @@ -356,10 +329,7 @@ XenNet_SendQueuedPackets(struct xennet_i 4.409 { 4.410 packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]); 4.411 //KdPrint((__DRIVER_NAME " Packet ready to send\n")); 4.412 - success = XenNet_HWSendPacket(xi, packet); 4.413 - if (success) 4.414 - InsertTailList(&xi->tx_sent_pkt_list, entry); 4.415 - else 4.416 + if (!XenNet_HWSendPacket(xi, packet)) 4.417 { 4.418 InsertHeadList(&xi->tx_waiting_pkt_list, entry); 4.419 break; 4.420 @@ -375,52 +345,15 @@ XenNet_SendQueuedPackets(struct xennet_i 4.421 //FUNCTION_EXIT(); 4.422 } 4.423 4.424 -// Called at <= DISPATCH_LEVEL with tx spinlock _NOT_ held 4.425 -static VOID 4.426 -XenNet_ReturnSentPackets(struct xennet_info *xi) 4.427 -{ 4.428 - PLIST_ENTRY entry; 4.429 - PNDIS_PACKET packets[32]; 4.430 - int packet_index = 0; 4.431 - int i = 0; 4.432 - KIRQL old_irql; 4.433 - 4.434 - //FUNCTION_ENTER(); 4.435 - 4.436 - old_irql = KeRaiseIrqlToDpcLevel(); 4.437 - KeAcquireSpinLockAtDpcLevel(&xi->tx_lock); 4.438 - entry = RemoveHeadList(&xi->tx_sent_pkt_list); 4.439 - 4.440 - while (entry != &xi->tx_sent_pkt_list) 4.441 - { 4.442 - packets[packet_index++] = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]); 4.443 - entry = RemoveHeadList(&xi->tx_sent_pkt_list); 4.444 - // try to minimize the need to acquire the spinlock repeatedly 4.445 - if (packet_index == 32 || entry == &xi->tx_sent_pkt_list) 4.446 - { 4.447 - KeReleaseSpinLockFromDpcLevel(&xi->tx_lock); 4.448 - for (i = 0; i < packet_index; i++) 4.449 - NdisMSendComplete(xi->adapter_handle, packets[i], NDIS_GET_PACKET_STATUS(packets[i])); 4.450 - if (entry != &xi->tx_sent_pkt_list) /* don't acquire the lock if we have no more packets to SendComplete */ 4.451 - KeAcquireSpinLockAtDpcLevel(&xi->tx_lock); 4.452 - packet_index = 0; 4.453 - } 4.454 - } 4.455 - if (!i) /* i will be == 0 if we didn't SendComplete any packets, and thus we will still have the lock */ 4.456 - KeReleaseSpinLockFromDpcLevel(&xi->tx_lock); 4.457 - KeLowerIrql(old_irql); 4.458 - //FUNCTION_EXIT(); 4.459 -} 4.460 - 4.461 +//ULONG packets_outstanding = 0; 4.462 // Called at DISPATCH_LEVEL 4.463 -//NDIS_STATUS 4.464 -//XenNet_TxBufferGC(struct xennet_info *xi) 4.465 VOID 4.466 XenNet_TxBufferGC(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2) 4.467 { 4.468 struct xennet_info *xi = context; 4.469 RING_IDX cons, prod; 4.470 - unsigned short id; 4.471 + PNDIS_PACKET head = NULL, tail = NULL; 4.472 + PNDIS_PACKET packet; 4.473 4.474 UNREFERENCED_PARAMETER(dpc); 4.475 UNREFERENCED_PARAMETER(arg1); 4.476 @@ -440,23 +373,28 @@ XenNet_TxBufferGC(PKDPC dpc, PVOID conte 4.477 for (cons = xi->tx.rsp_cons; cons != prod; cons++) 4.478 { 4.479 struct netif_tx_response *txrsp; 4.480 - 4.481 txrsp = RING_GET_RESPONSE(&xi->tx, cons); 4.482 - if (txrsp->status == NETIF_RSP_NULL) 4.483 - { 4.484 - put_no_id_on_freelist(xi); 4.485 - continue; // This would be the response to an extra_info packet 4.486 - } 4.487 + if (txrsp->status == NETIF_RSP_NULL || txrsp->id == 0xFFFF) 4.488 + continue; 4.489 4.490 - id = txrsp->id; 4.491 - 4.492 - if (xi->tx_mdls[id]) 4.493 + if (xi->tx_shadows[txrsp->id].sb) 4.494 { 4.495 - NdisAdjustBufferLength(xi->tx_mdls[id], PAGE_SIZE); 4.496 - XenFreelist_PutPage(&xi->tx_freelist, xi->tx_mdls[id]); 4.497 - xi->tx_mdls[id] = NULL; 4.498 + put_sb_on_freelist(xi, xi->tx_shadows[txrsp->id].sb); 4.499 + xi->tx_shadows[txrsp->id].sb = NULL; 4.500 } 4.501 - put_id_on_freelist(xi, id); 4.502 + 4.503 + if (xi->tx_shadows[txrsp->id].packet) 4.504 + { 4.505 + packet = xi->tx_shadows[txrsp->id].packet; 4.506 + *(PNDIS_PACKET *)&packet->MiniportReservedEx[0] = NULL; 4.507 + if (head) 4.508 + *(PNDIS_PACKET *)&tail->MiniportReservedEx[0] = packet; 4.509 + else 4.510 + head = packet; 4.511 + tail = packet; 4.512 + xi->tx_shadows[txrsp->id].packet = NULL; 4.513 + } 4.514 + put_id_on_freelist(xi, txrsp->id); 4.515 } 4.516 4.517 xi->tx.rsp_cons = prod; 4.518 @@ -469,8 +407,14 @@ XenNet_TxBufferGC(PKDPC dpc, PVOID conte 4.519 4.520 KeReleaseSpinLockFromDpcLevel(&xi->tx_lock); 4.521 4.522 - XenNet_ReturnSentPackets(xi); 4.523 - 4.524 + while (head) 4.525 + { 4.526 + packet = (PNDIS_PACKET)head; 4.527 +//KdPrint(("-packet = %p\n", packet)); 4.528 + head = *(PNDIS_PACKET *)&packet->MiniportReservedEx[0]; 4.529 + NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_SUCCESS); 4.530 + } 4.531 +//KdPrint((__DRIVER_NAME " packets_outstanding = %d\n", packets_outstanding)); 4.532 //FUNCTION_EXIT(); 4.533 } 4.534 4.535 @@ -494,8 +438,7 @@ XenNet_SendPackets( 4.536 { 4.537 for (i = 0; i < NumberOfPackets; i++) 4.538 { 4.539 - NDIS_SET_PACKET_STATUS(PacketArray[i], NDIS_STATUS_FAILURE); 4.540 - NdisMSendComplete(xi->adapter_handle, PacketArray[i], NDIS_GET_PACKET_STATUS(PacketArray[i])); 4.541 + NdisMSendComplete(xi->adapter_handle, PacketArray[i], NDIS_STATUS_FAILURE); 4.542 } 4.543 return; 4.544 } 4.545 @@ -506,6 +449,8 @@ XenNet_SendPackets( 4.546 for (i = 0; i < NumberOfPackets; i++) 4.547 { 4.548 packet = PacketArray[i]; 4.549 +//packets_outstanding++; 4.550 +//KdPrint(("+packet = %p\n", packet)); 4.551 ASSERT(packet); 4.552 *(ULONG *)&packet->MiniportReservedEx = 0; 4.553 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)]; 4.554 @@ -517,14 +462,14 @@ XenNet_SendPackets( 4.555 4.556 KeReleaseSpinLock(&xi->tx_lock, OldIrql); 4.557 4.558 - XenNet_ReturnSentPackets(xi); 4.559 - 4.560 //FUNCTION_EXIT(); 4.561 } 4.562 4.563 VOID 4.564 XenNet_TxResumeStart(xennet_info_t *xi) 4.565 { 4.566 + UNREFERENCED_PARAMETER(xi); 4.567 +#if 0 4.568 int i; 4.569 KIRQL old_irql; 4.570 4.571 @@ -538,12 +483,12 @@ XenNet_TxResumeStart(xennet_info_t *xi) 4.572 xi->tx_mdls[i] = NULL; 4.573 } 4.574 } 4.575 - XenFreelist_ResumeStart(&xi->tx_freelist); 4.576 xi->tx_id_free = 0; 4.577 xi->tx_no_id_used = 0; 4.578 for (i = 0; i < NET_TX_RING_SIZE; i++) 4.579 put_id_on_freelist(xi, (USHORT)i); 4.580 KeReleaseSpinLock(&xi->tx_lock, old_irql); 4.581 +#endif 4.582 } 4.583 4.584 VOID 4.585 @@ -552,39 +497,60 @@ XenNet_TxResumeEnd(xennet_info_t *xi) 4.586 KIRQL old_irql; 4.587 4.588 KeAcquireSpinLock(&xi->tx_lock, &old_irql); 4.589 - XenFreelist_ResumeEnd(&xi->tx_freelist); 4.590 XenNet_SendQueuedPackets(xi); 4.591 KeReleaseSpinLock(&xi->tx_lock, old_irql); 4.592 - XenNet_ReturnSentPackets(xi); 4.593 } 4.594 4.595 BOOLEAN 4.596 XenNet_TxInit(xennet_info_t *xi) 4.597 { 4.598 - USHORT i; 4.599 + NDIS_STATUS status; 4.600 + USHORT i, j; 4.601 4.602 KeInitializeSpinLock(&xi->tx_lock); 4.603 KeInitializeDpc(&xi->tx_dpc, XenNet_TxBufferGC, xi); 4.604 /* dpcs are only serialised to a single processor */ 4.605 KeSetTargetProcessorDpc(&xi->tx_dpc, 0); 4.606 //KeSetImportanceDpc(&xi->tx_dpc, HighImportance); 4.607 + InitializeListHead(&xi->tx_waiting_pkt_list); 4.608 + 4.609 + NdisAllocateBufferPool(&status, &xi->tx_buffer_pool, TX_SHARED_BUFFERS); 4.610 + if (status != NDIS_STATUS_SUCCESS) 4.611 + { 4.612 + KdPrint(("NdisAllocateBufferPool failed with 0x%x\n", status)); 4.613 + return FALSE; 4.614 + } 4.615 + 4.616 + xi->tx_ring_free = NET_TX_RING_SIZE; 4.617 + 4.618 + for (i = 0; i < TX_SHARED_BUFFERS / (PAGE_SIZE / TX_SHARED_BUFFER_SIZE); i++) 4.619 + { 4.620 + PVOID virtual; 4.621 + NDIS_PHYSICAL_ADDRESS logical; 4.622 + NdisMAllocateSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, &virtual, &logical); 4.623 + KdPrint((__DRIVER_NAME " Allocated SharedMemory at %p\n", virtual)); 4.624 + for (j = 0; j < PAGE_SIZE / TX_SHARED_BUFFER_SIZE; j++) 4.625 + { 4.626 + ULONG index = i * (PAGE_SIZE / TX_SHARED_BUFFER_SIZE) + j; 4.627 + xi->tx_sbs[index].id = index; 4.628 + xi->tx_sbs[index].virtual = (PUCHAR)virtual + j * TX_SHARED_BUFFER_SIZE; 4.629 + xi->tx_sbs[index].logical.QuadPart = logical.QuadPart + j * TX_SHARED_BUFFER_SIZE; 4.630 + put_sb_on_freelist(xi, &xi->tx_sbs[index]); 4.631 + } 4.632 + } 4.633 4.634 xi->tx_id_free = 0; 4.635 - xi->tx_no_id_used = 0; 4.636 for (i = 0; i < NET_TX_RING_SIZE; i++) 4.637 { 4.638 put_id_on_freelist(xi, i); 4.639 } 4.640 4.641 - XenFreelist_Init(xi, &xi->tx_freelist, &xi->tx_lock); 4.642 - 4.643 return TRUE; 4.644 } 4.645 4.646 /* 4.647 The ring is completely closed down now. We just need to empty anything left 4.648 -on our freelists and harvest anything left on the rings. The freelist timer 4.649 -will still be running though. 4.650 +on our freelists and harvest anything left on the rings. 4.651 */ 4.652 4.653 BOOLEAN 4.654 @@ -592,8 +558,8 @@ XenNet_TxShutdown(xennet_info_t *xi) 4.655 { 4.656 PLIST_ENTRY entry; 4.657 PNDIS_PACKET packet; 4.658 - PMDL mdl; 4.659 - ULONG i; 4.660 + //PMDL mdl; 4.661 + //ULONG i; 4.662 KIRQL OldIrql; 4.663 4.664 FUNCTION_ENTER(); 4.665 @@ -610,7 +576,8 @@ XenNet_TxShutdown(xennet_info_t *xi) 4.666 NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_FAILURE); 4.667 entry = RemoveHeadList(&xi->tx_waiting_pkt_list); 4.668 } 4.669 - 4.670 + 4.671 +#if 0 4.672 /* free sent-but-not-completed packets */ 4.673 for (i = 0; i < NET_TX_RING_SIZE; i++) 4.674 { 4.675 @@ -618,11 +585,11 @@ XenNet_TxShutdown(xennet_info_t *xi) 4.676 if (mdl) 4.677 { 4.678 NdisAdjustBufferLength(xi->tx_mdls[i], PAGE_SIZE); 4.679 - XenFreelist_PutPage(&xi->tx_freelist, xi->tx_mdls[i]); 4.680 + //XenFreelist_PutPage(&xi->tx_freelist, xi->tx_mdls[i]); 4.681 } 4.682 } 4.683 - 4.684 - XenFreelist_Dispose(&xi->tx_freelist); 4.685 +#endif 4.686 + NdisFreeBufferPool(xi->tx_buffer_pool); 4.687 4.688 KeReleaseSpinLock(&xi->tx_lock, OldIrql); 4.689