win-pvdrivers
changeset 532:b7491d4ebb3c
reduced copying in the rx path - ~30-50% rx performance increase
author | James Harper <james.harper@bendigoit.com.au> |
---|---|
date | Thu Jan 22 09:39:51 2009 +1100 (2009-01-22) |
parents | f966a0d692bc |
children | 82f26efb764f |
files | xennet/xennet.h xennet/xennet_common.c xennet/xennet_rx.c |
line diff
1.1 --- a/xennet/xennet.h Thu Jan 22 08:17:31 2009 +1100 1.2 +++ b/xennet/xennet.h Thu Jan 22 09:39:51 2009 +1100 1.3 @@ -172,12 +172,19 @@ SET_NET_ULONG(PVOID ptr, ULONG data) 1.4 #define RX_DFL_MIN_TARGET 256 1.5 #define RX_MAX_TARGET min(NET_RX_RING_SIZE, 256) 1.6 1.7 -#define MAX_BUFFERS_PER_PACKET 128 1.8 +#define MAX_BUFFERS_PER_PACKET NET_RX_RING_SIZE 1.9 + 1.10 +typedef struct { 1.11 + ULONG reference_count; 1.12 + PNDIS_BUFFER mdls[MAX_BUFFERS_PER_PACKET]; 1.13 + ULONG mdl_count; 1.14 +} mdl_alloc_t; 1.15 1.16 typedef struct { 1.17 PNDIS_BUFFER mdls[MAX_BUFFERS_PER_PACKET]; 1.18 + UCHAR header_data[132]; /* maximum possible size of ETH + IP + TCP/UDP headers */ 1.19 ULONG mdl_count; 1.20 - USHORT curr_mdl; 1.21 + USHORT curr_mdl_index; 1.22 USHORT curr_mdl_offset; 1.23 USHORT mss; 1.24 NDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info; 1.25 @@ -186,6 +193,8 @@ typedef struct { 1.26 BOOLEAN split_required; 1.27 UCHAR ip_version; 1.28 PUCHAR header; 1.29 + ULONG first_buffer_length; 1.30 + ULONG header_length; 1.31 UCHAR ip_proto; 1.32 USHORT total_length; 1.33 USHORT ip4_header_length; 1.34 @@ -382,10 +391,11 @@ XenNet_SetInformation( 1.35 #define PARSE_TOO_SMALL 1 /* first buffer is too small */ 1.36 #define PARSE_UNKNOWN_TYPE 2 1.37 1.38 +BOOLEAN 1.39 +XenNet_IncreasePacketHeader(packet_info_t *pi, ULONG new_header_size); 1.40 + 1.41 ULONG 1.42 -XenNet_ParsePacketHeader( 1.43 - packet_info_t *pi 1.44 -); 1.45 +XenNet_ParsePacketHeader(packet_info_t *pi); 1.46 1.47 VOID 1.48 XenNet_SumIpHeader( 1.49 @@ -406,7 +416,7 @@ XenNet_GetData( 1.50 PUSHORT length 1.51 ) 1.52 { 1.53 - PNDIS_BUFFER mdl = pi->mdls[pi->curr_mdl]; 1.54 + PNDIS_BUFFER mdl = pi->mdls[pi->curr_mdl_index]; 1.55 PUCHAR buffer = (PUCHAR)MmGetMdlVirtualAddress(mdl) + pi->curr_mdl_offset; 1.56 1.57 *length = (USHORT)min(req_length, MmGetMdlByteCount(mdl) - pi->curr_mdl_offset); 1.58 @@ -414,7 +424,7 @@ XenNet_GetData( 1.59 pi->curr_mdl_offset = pi->curr_mdl_offset + *length; 1.60 if (pi->curr_mdl_offset == MmGetMdlByteCount(mdl)) 1.61 { 1.62 - pi->curr_mdl++; 1.63 + pi->curr_mdl_index++; 1.64 pi->curr_mdl_offset = 0; 1.65 } 1.66 1.67 @@ -432,7 +442,7 @@ XenNet_ClearPacketInfo(packet_info_t *pi 1.68 #endif 1.69 #else 1.70 pi->mdl_count = 0; 1.71 - pi->curr_mdl = pi->curr_mdl_offset = 0; 1.72 + pi->curr_mdl_index = pi->curr_mdl_offset = 0; 1.73 pi->extra_info = pi->more_frags = pi->csum_blank = 1.74 pi->data_validated = pi->split_required = 0; 1.75 #endif
2.1 --- a/xennet/xennet_common.c Thu Jan 22 08:17:31 2009 +1100 2.2 +++ b/xennet/xennet_common.c Thu Jan 22 09:39:51 2009 +1100 2.3 @@ -20,32 +20,112 @@ Foundation, Inc., 51 Franklin Street, Fi 2.4 2.5 #include "xennet.h" 2.6 2.7 +/* 2.8 +Increase the header to a certain size 2.9 +*/ 2.10 + 2.11 +BOOLEAN 2.12 +XenNet_BuildHeader(packet_info_t *pi, ULONG new_header_size) 2.13 +{ 2.14 + ULONG bytes_remaining; 2.15 + PMDL current_mdl; 2.16 + 2.17 + //FUNCTION_ENTER(); 2.18 + 2.19 + if (new_header_size <= pi->header_length) 2.20 + { 2.21 + //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size < pi->header_length\n")); 2.22 + return TRUE; 2.23 + } 2.24 + 2.25 + if (new_header_size > ARRAY_SIZE(pi->header_data)) 2.26 + { 2.27 + //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size > ARRAY_SIZE(pi->header_data)\n")); 2.28 + return FALSE; 2.29 + } 2.30 + 2.31 + if (new_header_size <= pi->first_buffer_length) 2.32 + { 2.33 + //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " new_header_size <= pi->first_buffer_length\n")); 2.34 + pi->header_length = new_header_size; 2.35 + return TRUE; 2.36 + } 2.37 + else if (pi->header != pi->header_data) 2.38 + { 2.39 + //KdPrint((__DRIVER_NAME " Using header_data\n")); 2.40 + memcpy(pi->header_data, pi->header, pi->header_length); 2.41 + pi->header = pi->header_data; 2.42 + } 2.43 + 2.44 + bytes_remaining = new_header_size - pi->header_length; 2.45 + 2.46 + //KdPrint((__DRIVER_NAME " A bytes_remaining = %d, pi->curr_mdl_index = %d, pi->mdl_count = %d\n", 2.47 + // bytes_remaining, pi->curr_mdl_index, pi->mdl_count)); 2.48 + while (bytes_remaining && pi->curr_mdl_index < pi->mdl_count) 2.49 + { 2.50 + ULONG copy_size; 2.51 + 2.52 + //KdPrint((__DRIVER_NAME " B bytes_remaining = %d, pi->curr_mdl_index = %d, pi->mdl_count = %d\n", 2.53 + // bytes_remaining, pi->curr_mdl_index, pi->mdl_count)); 2.54 + current_mdl = pi->mdls[pi->curr_mdl_index]; 2.55 + if (MmGetMdlByteCount(current_mdl)) 2.56 + { 2.57 + copy_size = min(bytes_remaining, MmGetMdlByteCount(current_mdl) - pi->curr_mdl_offset); 2.58 + //KdPrint((__DRIVER_NAME " B copy_size = %d\n", copy_size)); 2.59 + memcpy(pi->header + pi->header_length, 2.60 + (PUCHAR)MmGetMdlVirtualAddress(current_mdl) + pi->curr_mdl_offset, copy_size); 2.61 + pi->curr_mdl_offset += copy_size; 2.62 + pi->header_length += copy_size; 2.63 + bytes_remaining -= copy_size; 2.64 + } 2.65 + if (pi->curr_mdl_offset == MmGetMdlByteCount(current_mdl)) 2.66 + { 2.67 + pi->curr_mdl_index++; 2.68 + pi->curr_mdl_offset = 0; 2.69 + } 2.70 + } 2.71 + //KdPrint((__DRIVER_NAME " C bytes_remaining = %d, pi->curr_mdl_index = %d, pi->mdl_count = %d\n", 2.72 + // bytes_remaining, pi->curr_mdl_index, pi->mdl_count)); 2.73 + if (bytes_remaining) 2.74 + { 2.75 + //KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ " bytes_remaining\n")); 2.76 + return FALSE; 2.77 + } 2.78 + //FUNCTION_EXIT(); 2.79 + return TRUE; 2.80 +} 2.81 + 2.82 ULONG 2.83 -XenNet_ParsePacketHeader( 2.84 - packet_info_t *pi 2.85 -) 2.86 +XenNet_ParsePacketHeader(packet_info_t *pi) 2.87 { 2.88 - UINT header_length; 2.89 - 2.90 -// KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n")); 2.91 + //FUNCTION_ENTER(); 2.92 2.93 ASSERT(pi->mdls[0]); 2.94 2.95 - NdisQueryBufferSafe(pi->mdls[0], (PVOID) &pi->header, &header_length, NormalPagePriority); 2.96 + NdisQueryBufferSafe(pi->mdls[0], (PVOID) &pi->header, &pi->first_buffer_length, NormalPagePriority); 2.97 2.98 -// what about if the buffer isn't completely on one page??? 2.99 - if (ADDRESS_AND_SIZE_TO_SPAN_PAGES(MmGetMdlVirtualAddress(pi->mdls[0]), header_length) != 1) 2.100 - KdPrint((__DRIVER_NAME " header crosses a page!\n")); 2.101 - 2.102 - 2.103 - if (header_length < XN_HDR_SIZE + 20 + 20) // minimum size of first buffer is ETH + IP + TCP header 2.104 + pi->header_length = 0; 2.105 + pi->curr_mdl_index = 0; 2.106 + pi->curr_mdl_offset = 0; 2.107 + 2.108 + if (!XenNet_BuildHeader(pi, (ULONG)XN_HDR_SIZE)) 2.109 { 2.110 + KdPrint((__DRIVER_NAME " packet too small (Ethernet Header)\n")); 2.111 return PARSE_TOO_SMALL; 2.112 } 2.113 - 2.114 + 2.115 switch (GET_NET_PUSHORT(&pi->header[12])) // L2 protocol field 2.116 { 2.117 case 0x0800: 2.118 + //KdPrint((__DRIVER_NAME " IP\n")); 2.119 + if (pi->header_length < (ULONG)(XN_HDR_SIZE + 20)) 2.120 + { 2.121 + if (!XenNet_BuildHeader(pi, (ULONG)(XN_HDR_SIZE + 20))) 2.122 + { 2.123 + KdPrint((__DRIVER_NAME " packet too small (IP Header)\n")); 2.124 + return PARSE_TOO_SMALL; 2.125 + } 2.126 + } 2.127 pi->ip_version = (pi->header[XN_HDR_SIZE + 0] & 0xF0) >> 4; 2.128 if (pi->ip_version != 4) 2.129 { 2.130 @@ -53,21 +133,25 @@ XenNet_ParsePacketHeader( 2.131 return PARSE_UNKNOWN_TYPE; 2.132 } 2.133 pi->ip4_header_length = (pi->header[XN_HDR_SIZE + 0] & 0x0F) << 2; 2.134 - if (header_length < (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + 20)) 2.135 + if (pi->header_length < (ULONG)(XN_HDR_SIZE + 20 + pi->ip4_header_length)) 2.136 { 2.137 - KdPrint((__DRIVER_NAME " first buffer is only %d bytes long, must be >= %d (1)\n", header_length, (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + 20))); 2.138 + if (!XenNet_BuildHeader(pi, (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + 20))) 2.139 + { 2.140 + KdPrint((__DRIVER_NAME " packet too small (IP Header + IP Options + TCP Header)\n")); 2.141 + return PARSE_TOO_SMALL; 2.142 + } 2.143 #if 0 2.144 + KdPrint((__DRIVER_NAME " first buffer is only %d bytes long, must be >= %d (1)\n", pi->header_length, (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + 20))); 2.145 KdPrint((__DRIVER_NAME " total_length = %d\n", pi->total_length)); 2.146 for (i = 0; i < pi->mdl_count; i++) 2.147 { 2.148 KdPrint((__DRIVER_NAME " mdl %d length = %d\n", i, MmGetMdlByteCount(pi->mdls[i]))); 2.149 } 2.150 #endif 2.151 - return PARSE_TOO_SMALL; 2.152 } 2.153 break; 2.154 default: 2.155 -// KdPrint((__DRIVER_NAME " Not IP\n")); 2.156 + KdPrint((__DRIVER_NAME " Not IP (%d)\n", GET_NET_PUSHORT(&pi->header[12]))); 2.157 return PARSE_UNKNOWN_TYPE; 2.158 } 2.159 pi->ip_proto = pi->header[XN_HDR_SIZE + 9]; 2.160 @@ -77,15 +161,23 @@ XenNet_ParsePacketHeader( 2.161 case 17: // UDP 2.162 break; 2.163 default: 2.164 + KdPrint((__DRIVER_NAME " Not TCP/UDP (%d)\n", pi->ip_proto)); 2.165 return PARSE_UNKNOWN_TYPE; 2.166 } 2.167 pi->ip4_length = GET_NET_PUSHORT(&pi->header[XN_HDR_SIZE + 2]); 2.168 pi->tcp_header_length = (pi->header[XN_HDR_SIZE + pi->ip4_header_length + 12] & 0xf0) >> 2; 2.169 2.170 - if (header_length < (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + 20)) // pi->tcp_header_length)) 2.171 + if (pi->header_length < (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + pi->tcp_header_length)) 2.172 { 2.173 - KdPrint((__DRIVER_NAME " first buffer is only %d bytes long, must be >= %d (2)\n", header_length, (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + pi->tcp_header_length))); 2.174 + if (!XenNet_BuildHeader(pi, (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + 20))) 2.175 + { 2.176 + KdPrint((__DRIVER_NAME " packet too small (IP Header + IP Options + TCP Header + TCP Options)\n")); 2.177 + return PARSE_TOO_SMALL; 2.178 + } 2.179 +#if 0 2.180 + KdPrint((__DRIVER_NAME " first buffer is only %d bytes long, must be >= %d (2)\n", pi->header_length, (ULONG)(XN_HDR_SIZE + pi->ip4_header_length + pi->tcp_header_length))); 2.181 return PARSE_TOO_SMALL; 2.182 +#endif 2.183 } 2.184 2.185 pi->tcp_length = pi->ip4_length - pi->ip4_header_length - pi->tcp_header_length; 2.186 @@ -94,7 +186,11 @@ XenNet_ParsePacketHeader( 2.187 pi->tcp_has_options = (BOOLEAN)(pi->tcp_header_length > 20); 2.188 if (pi->mss > 0 && pi->tcp_length > pi->mss) 2.189 pi->split_required = TRUE; 2.190 -// KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ "\n")); 2.191 + 2.192 + //KdPrint((__DRIVER_NAME " ip4_length = %d\n", pi->ip4_length)); 2.193 + //KdPrint((__DRIVER_NAME " tcp_length = %d\n", pi->tcp_length)); 2.194 + //FUNCTION_EXIT(); 2.195 + 2.196 return PARSE_OK; 2.197 } 2.198
3.1 --- a/xennet/xennet_rx.c Thu Jan 22 08:17:31 2009 +1100 3.2 +++ b/xennet/xennet_rx.c Thu Jan 22 09:39:51 2009 +1100 3.3 @@ -188,7 +188,7 @@ XenNet_MakePacket(struct xennet_info *xi 3.4 xi->rxpi.tcp_remaining = xi->rxpi.tcp_remaining - out_remaining; 3.5 do 3.6 { 3.7 - ASSERT(xi->rxpi.curr_mdl < xi->rxpi.mdl_count); 3.8 + ASSERT(xi->rxpi.curr_mdl_index < xi->rxpi.mdl_count); 3.9 in_buffer = XenNet_GetData(&xi->rxpi, out_remaining, &length); 3.10 memcpy(&out_buffer[out_offset], in_buffer, length); 3.11 out_remaining = out_remaining - length; 3.12 @@ -437,10 +437,12 @@ XenNet_MakePackets( 3.13 } 3.14 3.15 xi->rxpi.tcp_remaining = xi->rxpi.tcp_length; 3.16 +#if 0 // _index and _offset set by ParseHeader 3.17 if (MmGetMdlByteCount(xi->rxpi.mdls[0]) > (ULONG)(XN_HDR_SIZE + xi->rxpi.ip4_header_length + xi->rxpi.tcp_header_length)) 3.18 xi->rxpi.curr_mdl_offset = XN_HDR_SIZE + xi->rxpi.ip4_header_length + xi->rxpi.tcp_header_length; 3.19 else 3.20 - xi->rxpi.curr_mdl = 1; 3.21 + xi->rxpi.curr_mdl_index = 1; 3.22 +#endif 3.23 3.24 /* we can make certain assumptions here as the following code is only for tcp4 */ 3.25 psh = xi->rxpi.header[XN_HDR_SIZE + xi->rxpi.ip4_header_length + 13] & 8; 3.26 @@ -485,6 +487,7 @@ done: 3.27 for (i = 0; i < xi->rxpi.mdl_count; i++) 3.28 { 3.29 NdisAdjustBufferLength(xi->rxpi.mdls[i], PAGE_SIZE); 3.30 + xi->rxpi.mdls[i]->ByteOffset = 0; 3.31 XenFreelist_PutPage(&xi->rx_freelist, xi->rxpi.mdls[i]); 3.32 } 3.33 XenNet_ClearPacketInfo(&xi->rxpi); 3.34 @@ -534,8 +537,6 @@ XenNet_RxTimerDpc(PKDPC dpc, PVOID conte 3.35 } 3.36 3.37 // Called at DISPATCH_LEVEL 3.38 -//NDIS_STATUS 3.39 -//XenNet_RxBufferCheck(struct xennet_info *xi, BOOLEAN is_timer) 3.40 static VOID 3.41 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2) 3.42 { 3.43 @@ -630,6 +631,9 @@ XenNet_RxBufferCheck(PKDPC dpc, PVOID co 3.44 xi->rxpi.data_validated = TRUE; 3.45 } 3.46 3.47 + NdisAdjustBufferLength(mdl, rxrsp->status); 3.48 + xi->rxpi.mdls[xi->rxpi.mdl_count++] = mdl; 3.49 +#if 0 3.50 if (!xi->rxpi.mdl_count || MmGetMdlByteCount(xi->rxpi.mdls[xi->rxpi.mdl_count - 1]) == PAGE_SIZE) 3.51 { 3.52 /* first buffer or no room in current buffer */ 3.53 @@ -664,6 +668,7 @@ XenNet_RxBufferCheck(PKDPC dpc, PVOID co 3.54 NdisAdjustBufferLength(mdl, copy_size); 3.55 xi->rxpi.mdls[xi->rxpi.mdl_count++] = mdl; 3.56 } 3.57 +#endif 3.58 xi->rxpi.extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info); 3.59 xi->rxpi.more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data); 3.60 xi->rxpi.total_length = xi->rxpi.total_length + rxrsp->status; 3.61 @@ -778,6 +783,7 @@ XenNet_ReturnPacket( 3.62 while (mdl) 3.63 { 3.64 //KdPrint((__DRIVER_NAME " packet = %p, mdl = %p\n", Packet, mdl)); 3.65 + mdl->ByteOffset = 0; 3.66 NdisAdjustBufferLength(mdl, PAGE_SIZE); 3.67 XenFreelist_PutPage(&xi->rx_freelist, mdl); 3.68 NdisUnchainBufferAtBack(Packet, &mdl);