win-pvdrivers

annotate xennet/xennet_rx.c @ 1023:1ce315b193d1

Change all NT_ASSERT to XN_ASSERT
author James Harper <james.harper@bendigoit.com.au>
date Tue Feb 19 15:12:35 2013 +1100 (2013-02-19)
parents 9b6213b6be25
children cb767700f91c
rev   line source
james@1007 1 /*
james@1007 2 PV Net Driver for Windows Xen HVM Domains
james@1007 3 Copyright (C) 2007 James Harper
james@1007 4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
james@1007 5
james@1007 6 This program is free software; you can redistribute it and/or
james@1007 7 modify it under the terms of the GNU General Public License
james@1007 8 as published by the Free Software Foundation; either version 2
james@1007 9 of the License, or (at your option) any later version.
james@1007 10
james@1007 11 This program is distributed in the hope that it will be useful,
james@1007 12 but WITHOUT ANY WARRANTY; without even the implied warranty of
james@1007 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
james@1007 14 GNU General Public License for more details.
james@1007 15
james@1007 16 You should have received a copy of the GNU General Public License
james@1007 17 along with this program; if not, write to the Free Software
james@1007 18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
james@1007 19 */
james@1007 20
james@1007 21 #include "xennet.h"
james@1007 22
james@1007 23 static __inline shared_buffer_t *
james@1007 24 get_pb_from_freelist(struct xennet_info *xi)
james@1007 25 {
james@1007 26 shared_buffer_t *pb;
james@1007 27 PVOID ptr_ref;
james@1007 28
james@1007 29 if (stack_pop(xi->rx_pb_stack, &ptr_ref))
james@1007 30 {
james@1007 31 pb = ptr_ref;
james@1007 32 pb->ref_count = 1;
james@1007 33 InterlockedDecrement(&xi->rx_pb_free);
james@1007 34 return pb;
james@1007 35 }
james@1007 36
james@1007 37 /* don't allocate a new one if we are shutting down */
james@1007 38 if (xi->device_state != DEVICE_STATE_INITIALISING && xi->device_state != DEVICE_STATE_ACTIVE)
james@1007 39 return NULL;
james@1007 40
james@1007 41 pb = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(shared_buffer_t), XENNET_POOL_TAG, LowPoolPriority);
james@1007 42 if (!pb)
james@1007 43 return NULL;
james@1007 44 pb->virtual = ExAllocatePoolWithTagPriority(NonPagedPool, PAGE_SIZE, XENNET_POOL_TAG, LowPoolPriority);
james@1007 45 if (!pb->virtual)
james@1007 46 {
james@1007 47 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
james@1007 48 return NULL;
james@1007 49 }
james@1007 50 pb->mdl = IoAllocateMdl(pb->virtual, PAGE_SIZE, FALSE, FALSE, NULL);
james@1007 51 if (!pb->mdl)
james@1007 52 {
james@1007 53 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
james@1007 54 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
james@1007 55 return NULL;
james@1007 56 }
james@1007 57 pb->gref = (grant_ref_t)XnGrantAccess(xi->handle,
james@1007 58 (ULONG)(MmGetPhysicalAddress(pb->virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
james@1007 59 if (pb->gref == INVALID_GRANT_REF)
james@1007 60 {
james@1007 61 IoFreeMdl(pb->mdl);
james@1007 62 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
james@1007 63 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
james@1007 64 return NULL;
james@1007 65 }
james@1007 66 MmBuildMdlForNonPagedPool(pb->mdl);
james@1007 67 pb->ref_count = 1;
james@1007 68 return pb;
james@1007 69 }
james@1007 70
james@1007 71 static __inline VOID
james@1007 72 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
james@1007 73 {
james@1007 74 UNREFERENCED_PARAMETER(xi);
james@1007 75 InterlockedIncrement(&pb->ref_count);
james@1007 76 }
james@1007 77
james@1007 78 static __inline VOID
james@1007 79 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
james@1007 80 {
james@1007 81 if (InterlockedDecrement(&pb->ref_count) == 0)
james@1007 82 {
james@1007 83 //NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
james@1007 84 //NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
james@1007 85 if (xi->rx_pb_free > RX_MAX_PB_FREELIST)
james@1007 86 {
james@1007 87 XnEndAccess(xi->handle, pb->gref, FALSE, (ULONG)'XNRX');
james@1007 88 IoFreeMdl(pb->mdl);
james@1007 89 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
james@1007 90 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
james@1007 91 return;
james@1007 92 }
james@1007 93 pb->mdl->ByteCount = PAGE_SIZE;
james@1007 94 pb->mdl->Next = NULL;
james@1007 95 pb->next = NULL;
james@1007 96 stack_push(xi->rx_pb_stack, pb);
james@1007 97 InterlockedIncrement(&xi->rx_pb_free);
james@1007 98 }
james@1007 99 }
james@1007 100
james@1007 101 static __inline shared_buffer_t *
james@1007 102 get_hb_from_freelist(struct xennet_info *xi)
james@1007 103 {
james@1007 104 shared_buffer_t *hb;
james@1007 105 PVOID ptr_ref;
james@1007 106
james@1007 107 if (stack_pop(xi->rx_hb_stack, &ptr_ref))
james@1007 108 {
james@1007 109 hb = ptr_ref;
james@1007 110 InterlockedDecrement(&xi->rx_hb_free);
james@1007 111 return hb;
james@1007 112 }
james@1007 113
james@1007 114 /* don't allocate a new one if we are shutting down */
james@1007 115 if (xi->device_state != DEVICE_STATE_INITIALISING && xi->device_state != DEVICE_STATE_ACTIVE)
james@1007 116 return NULL;
james@1007 117
james@1007 118 hb = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, XENNET_POOL_TAG, LowPoolPriority);
james@1007 119 if (!hb)
james@1007 120 return NULL;
james@1007 121 NdisZeroMemory(hb, sizeof(shared_buffer_t));
james@1007 122 hb->mdl = IoAllocateMdl(hb + 1, MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, FALSE, FALSE, NULL);
james@1007 123 if (!hb->mdl) {
james@1007 124 ExFreePoolWithTag(hb, XENNET_POOL_TAG);
james@1007 125 return NULL;
james@1007 126 }
james@1007 127 MmBuildMdlForNonPagedPool(hb->mdl);
james@1007 128 return hb;
james@1007 129 }
james@1007 130
james@1007 131 static __inline VOID
james@1007 132 put_hb_on_freelist(struct xennet_info *xi, shared_buffer_t *hb)
james@1007 133 {
james@1023 134 XN_ASSERT(xi);
james@1007 135 hb->mdl->ByteCount = sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH;
james@1007 136 hb->mdl->Next = NULL;
james@1007 137 hb->next = NULL;
james@1007 138 stack_push(xi->rx_hb_stack, hb);
james@1007 139 InterlockedIncrement(&xi->rx_hb_free);
james@1007 140 }
james@1007 141
james@1007 142 // Called at DISPATCH_LEVEL with rx lock held
james@1007 143 static VOID
james@1007 144 XenNet_FillRing(struct xennet_info *xi)
james@1007 145 {
james@1007 146 unsigned short id;
james@1007 147 shared_buffer_t *page_buf;
james@1007 148 ULONG i, notify;
james@1007 149 ULONG batch_target;
james@1007 150 RING_IDX req_prod = xi->rx_ring.req_prod_pvt;
james@1007 151 netif_rx_request_t *req;
james@1007 152
james@1007 153 //FUNCTION_ENTER();
james@1007 154
james@1007 155 if (xi->device_state != DEVICE_STATE_ACTIVE)
james@1007 156 return;
james@1007 157
james@1007 158 batch_target = xi->rx_target - (req_prod - xi->rx_ring.rsp_cons);
james@1007 159
james@1007 160 if (batch_target < (xi->rx_target >> 2)) {
james@1007 161 //FUNCTION_EXIT();
james@1007 162 return; /* only refill if we are less than 3/4 full already */
james@1007 163 }
james@1007 164
james@1007 165 for (i = 0; i < batch_target; i++) {
james@1007 166 page_buf = get_pb_from_freelist(xi);
james@1007 167 if (!page_buf) {
james@1007 168 KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
james@1007 169 break;
james@1007 170 }
james@1007 171 xi->rx_id_free--;
james@1007 172
james@1007 173 /* Give to netback */
james@1007 174 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
james@1023 175 XN_ASSERT(xi->rx_ring_pbs[id] == NULL);
james@1007 176 xi->rx_ring_pbs[id] = page_buf;
james@1007 177 req = RING_GET_REQUEST(&xi->rx_ring, req_prod + i);
james@1007 178 req->id = id;
james@1007 179 req->gref = page_buf->gref;
james@1023 180 XN_ASSERT(req->gref != INVALID_GRANT_REF);
james@1007 181 }
james@1007 182 KeMemoryBarrier();
james@1007 183 xi->rx_ring.req_prod_pvt = req_prod + i;
james@1007 184 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx_ring, notify);
james@1007 185 if (notify) {
james@1007 186 XnNotify(xi->handle, xi->event_channel);
james@1007 187 }
james@1007 188
james@1007 189 //FUNCTION_EXIT();
james@1007 190
james@1007 191 return;
james@1007 192 }
james@1007 193
james@1007 194 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 195 typedef struct {
james@1007 196 PNDIS_PACKET first_packet;
james@1007 197 PNDIS_PACKET last_packet;
james@1007 198 ULONG packet_count;
james@1007 199 } rx_context_t;
james@914 200 #else
james@1007 201 typedef struct {
james@1007 202 PNET_BUFFER_LIST first_nbl;
james@1007 203 PNET_BUFFER_LIST last_nbl;
james@1007 204 ULONG packet_count;
james@1007 205 ULONG nbl_count;
james@1007 206 } rx_context_t;
james@906 207 #endif
james@1007 208
james@1011 209 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 210 /*
james@1007 211 NDIS5 appears to insist that the checksum on received packets is correct, and won't
james@1007 212 believe us when we lie about it, which happens when the packet is generated on the
james@1007 213 same bridge in Dom0. Doh!
james@1007 214 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
james@1007 215 */
james@1007 216
james@1007 217 static BOOLEAN
james@1007 218 XenNet_SumPacketData(
james@1007 219 packet_info_t *pi,
james@1007 220 PNDIS_PACKET packet,
james@1007 221 BOOLEAN set_csum) {
james@1007 222 USHORT i;
james@1007 223 PUCHAR buffer;
james@1007 224 PMDL mdl;
james@1007 225 UINT total_length;
james@1007 226 UINT data_length;
james@1007 227 UINT buffer_length;
james@1007 228 USHORT buffer_offset;
james@1007 229 ULONG csum;
james@1007 230 PUSHORT csum_ptr;
james@1007 231 USHORT remaining;
james@1007 232 USHORT ip4_length;
james@1007 233 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
james@1007 234
james@1007 235 //FUNCTION_ENTER();
james@1007 236
james@1007 237 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
james@1007 238 if (!buffer) {
james@1007 239 FUNCTION_MSG("NdisGetFirstBufferFromPacketSafe failed, buffer == NULL\n");
james@1007 240 return FALSE;
james@1007 241 }
james@1023 242 XN_ASSERT(mdl);
james@1007 243
james@1007 244 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
james@1007 245 data_length = ip4_length + XN_HDR_SIZE;
james@1007 246
james@1007 247 if ((USHORT)data_length > total_length) {
james@1007 248 FUNCTION_MSG("Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length);
james@1007 249 return FALSE;
james@1007 250 }
james@1007 251
james@1007 252 switch (pi->ip_proto) {
james@1007 253 case 6:
james@1023 254 XN_ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
james@1007 255 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
james@1007 256 break;
james@1007 257 case 17:
james@1023 258 XN_ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
james@1007 259 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
james@1007 260 break;
james@1007 261 default:
james@1007 262 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
james@1007 263 //FUNCTION_EXIT();
james@1007 264 return FALSE; // should never happen
james@1007 265 }
james@1007 266
james@1007 267 if (set_csum)
james@1007 268 *csum_ptr = 0;
james@1007 269
james@1007 270 csum = 0;
james@1007 271 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
james@1007 272 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
james@1007 273 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
james@1007 274
james@1007 275 remaining = ip4_length - pi->ip4_header_length;
james@1007 276
james@1007 277 csum += remaining;
james@1007 278
james@1007 279 csum_span = FALSE;
james@1007 280 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
james@1007 281 while (i < data_length) {
james@1007 282 /* don't include the checksum field itself in the calculation */
james@1007 283 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6)) {
james@1007 284 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
james@1007 285 i += 2;
james@1007 286 buffer_offset += 2;
james@1007 287 continue;
james@1007 288 }
james@1007 289 if (csum_span) {
james@1007 290 /* the other half of the next bit */
james@1023 291 XN_ASSERT(buffer_offset == 0);
james@1007 292 csum += (USHORT)buffer[buffer_offset];
james@1007 293 csum_span = FALSE;
james@1007 294 i += 1;
james@1007 295 buffer_offset += 1;
james@1007 296 } else if (buffer_offset == buffer_length - 1) {
james@1007 297 /* deal with a buffer ending on an odd byte boundary */
james@1007 298 csum += (USHORT)buffer[buffer_offset] << 8;
james@1007 299 csum_span = TRUE;
james@1007 300 i += 1;
james@1007 301 buffer_offset += 1;
james@1007 302 } else {
james@1007 303 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
james@1007 304 i += 2;
james@1007 305 buffer_offset += 2;
james@1007 306 }
james@1007 307 if (buffer_offset == buffer_length && i < total_length) {
james@1007 308 NdisGetNextBuffer(mdl, &mdl);
james@1007 309 if (mdl == NULL) {
james@1007 310 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
james@1007 311 return FALSE; // should never happen
james@1007 312 }
james@1007 313 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
james@1023 314 XN_ASSERT(buffer_length);
james@1007 315 buffer_offset = 0;
james@1007 316 }
james@1007 317 }
james@1007 318
james@1007 319 while (csum & 0xFFFF0000)
james@1007 320 csum = (csum & 0xFFFF) + (csum >> 16);
james@1007 321
james@1007 322 if (set_csum) {
james@1007 323 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
james@1007 324 } else {
james@1007 325 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
james@1007 326 }
james@1007 327 return TRUE;
james@1007 328 }
james@1011 329 #endif
james@1007 330
james@1007 331 static BOOLEAN
james@1007 332 XenNet_MakePacket(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi) {
james@1011 333 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 334 NDIS_STATUS status;
james@1007 335 PNDIS_PACKET packet;
james@1007 336 #else
james@1007 337 PNET_BUFFER_LIST nbl;
james@1007 338 PNET_BUFFER packet;
james@1007 339 #endif
james@1007 340 PMDL mdl_head, mdl_tail, curr_mdl;
james@1007 341 PUCHAR header_va;
james@1007 342 ULONG out_remaining;
james@1007 343 ULONG header_extra;
james@1007 344 shared_buffer_t *header_buf;
james@1007 345 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 346 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
james@1007 347 UINT packet_length;
james@1007 348 #else
james@1007 349 NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csum_info;
james@1007 350 #endif
james@1007 351 //FUNCTION_ENTER();
james@1007 352
james@1007 353 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 354 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
james@1007 355 if (status != NDIS_STATUS_SUCCESS) {
james@1007 356 FUNCTION_MSG("No free packets\n");
james@1007 357 return FALSE;
james@1007 358 }
james@1007 359 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
james@1007 360 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
james@1007 361 #else
james@1007 362 nbl = NdisAllocateNetBufferList(xi->rx_nbl_pool, 0, 0);
james@1007 363 if (!nbl) {
james@1007 364 /* buffers will be freed in MakePackets */
james@1007 365 KdPrint((__DRIVER_NAME " No free nbls\n"));
james@1007 366 //FUNCTION_EXIT();
james@1007 367 return FALSE;
james@1007 368 }
james@1007 369
james@1007 370 packet = NdisAllocateNetBuffer(xi->rx_packet_pool, NULL, 0, 0);
james@1007 371 if (!packet) {
james@1007 372 KdPrint((__DRIVER_NAME " No free packets\n"));
james@1007 373 NdisFreeNetBufferList(nbl);
james@1007 374 //FUNCTION_EXIT();
james@1007 375 return FALSE;
james@1007 376 }
james@1007 377 #endif
james@1007 378
james@1007 379 if (!pi->first_mdl->Next && !pi->split_required) {
james@1007 380 /* a single buffer <= MTU */
james@1007 381 header_buf = NULL;
james@1007 382 XenNet_BuildHeader(pi, pi->first_mdl_virtual, pi->first_mdl_length);
james@1007 383 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 384 NdisChainBufferAtBack(packet, pi->first_mdl);
james@1007 385 PACKET_FIRST_PB(packet) = pi->first_pb;
james@1007 386 #else
james@1007 387 NET_BUFFER_FIRST_MDL(packet) = pi->first_mdl;
james@1007 388 NET_BUFFER_CURRENT_MDL(packet) = pi->first_mdl;
james@1007 389 NET_BUFFER_CURRENT_MDL_OFFSET(packet) = 0;
james@1007 390 NET_BUFFER_DATA_OFFSET(packet) = 0;
james@1007 391 NET_BUFFER_DATA_LENGTH(packet) = pi->total_length;
james@1007 392 NB_FIRST_PB(packet) = pi->first_pb;
james@1007 393 #endif
james@1007 394 ref_pb(xi, pi->first_pb);
james@1007 395 } else {
james@1023 396 XN_ASSERT(ndis_os_minor_version >= 1);
james@1007 397 header_buf = get_hb_from_freelist(xi);
james@1007 398 if (!header_buf) {
james@1007 399 FUNCTION_MSG("No free header buffers\n");
james@1007 400 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 401 NdisUnchainBufferAtFront(packet, &curr_mdl);
james@1007 402 NdisFreePacket(packet);
james@1007 403 #else
james@1007 404 NdisFreeNetBufferList(nbl);
james@1007 405 NdisFreeNetBuffer(packet);
james@1007 406 #endif
james@1007 407 return FALSE;
james@1007 408 }
james@1007 409 header_va = (PUCHAR)(header_buf + 1);
james@1007 410 NdisMoveMemory(header_va, pi->header, pi->header_length);
james@1007 411 //KdPrint((__DRIVER_NAME " header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead));
james@1007 412 //KdPrint((__DRIVER_NAME " ip4_header_length = %d\n", pi->ip4_header_length));
james@1007 413 //KdPrint((__DRIVER_NAME " tcp_header_length = %d\n", pi->tcp_header_length));
james@1007 414 /* make sure only the header is in the first buffer (or the entire packet, but that is done in the above case) */
james@1007 415 XenNet_BuildHeader(pi, header_va, MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
james@1007 416 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
james@1023 417 XN_ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
james@1007 418 header_buf->mdl->ByteCount = pi->header_length;
james@1007 419 mdl_head = mdl_tail = curr_mdl = header_buf->mdl;
james@1007 420 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 421 PACKET_FIRST_PB(packet) = header_buf;
james@1007 422 header_buf->next = pi->curr_pb;
james@1007 423 NdisChainBufferAtBack(packet, mdl_head);
james@1007 424 #else
james@1007 425 NB_FIRST_PB(packet) = header_buf;
james@1007 426 header_buf->next = pi->curr_pb;
james@1007 427 NET_BUFFER_FIRST_MDL(packet) = mdl_head;
james@1007 428 NET_BUFFER_CURRENT_MDL(packet) = mdl_head;
james@1007 429 NET_BUFFER_CURRENT_MDL_OFFSET(packet) = 0;
james@1007 430 NET_BUFFER_DATA_OFFSET(packet) = 0;
james@1007 431 NET_BUFFER_DATA_LENGTH(packet) = pi->header_length;
james@1007 432 #endif
james@1007 433
james@1007 434 if (pi->split_required) {
james@1007 435 ULONG tcp_length;
james@1007 436 USHORT new_ip4_length;
james@1007 437 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
james@1007 438 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
james@1007 439 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
james@1007 440 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
james@1007 441 pi->tcp_seq += tcp_length;
james@1007 442 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
james@1007 443 /* part of the packet is already present in the header buffer for lookahead */
james@1007 444 out_remaining = tcp_length - header_extra;
james@1023 445 XN_ASSERT((LONG)out_remaining >= 0);
james@1007 446 } else {
james@1007 447 out_remaining = pi->total_length - pi->header_length;
james@1023 448 XN_ASSERT((LONG)out_remaining >= 0);
james@1007 449 }
james@1007 450
james@1007 451 while (out_remaining != 0) {
james@1007 452 //ULONG in_buffer_offset;
james@1007 453 ULONG in_buffer_length;
james@1007 454 ULONG out_length;
james@1007 455
james@1007 456 //KdPrint((__DRIVER_NAME " in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb));
james@1007 457 if (!pi->curr_mdl || !pi->curr_pb) {
james@1007 458 KdPrint((__DRIVER_NAME " out of buffers for packet\n"));
james@1007 459 //KdPrint((__DRIVER_NAME " out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb));
james@1007 460 // TODO: free some stuff or we'll leak
james@1007 461 /* unchain buffers then free packet */
james@1007 462 //FUNCTION_EXIT();
james@1007 463 return FALSE;
james@1007 464 }
james@1007 465
james@1007 466 in_buffer_length = MmGetMdlByteCount(pi->curr_mdl);
james@1007 467 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
james@1007 468 curr_mdl = IoAllocateMdl((PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length, FALSE, FALSE, NULL);
james@1023 469 XN_ASSERT(curr_mdl);
james@1007 470 IoBuildPartialMdl(pi->curr_mdl, curr_mdl, (PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length);
james@1007 471 mdl_tail->Next = curr_mdl;
james@1007 472 mdl_tail = curr_mdl;
james@1007 473 curr_mdl->Next = NULL; /* I think this might be redundant */
james@1007 474 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 475 #else
james@1007 476 NET_BUFFER_DATA_LENGTH(packet) += out_length;
james@1007 477 #endif
james@1007 478 ref_pb(xi, pi->curr_pb);
james@1007 479 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
james@1007 480 if (pi->curr_mdl_offset == in_buffer_length) {
james@1007 481 pi->curr_mdl = pi->curr_mdl->Next;
james@1007 482 pi->curr_pb = pi->curr_pb->next;
james@1007 483 pi->curr_mdl_offset = 0;
james@1007 484 }
james@1007 485 out_remaining -= out_length;
james@1007 486 }
james@1007 487 if (pi->split_required) {
james@1007 488 // TODO: only if Ip checksum is disabled...
james@1007 489 //XenNet_SumIpHeader(header_va, pi->ip4_header_length);
james@1007 490 }
james@1007 491 if (header_extra > 0)
james@1007 492 pi->header_length -= header_extra;
james@1007 493 }
james@1007 494
james@1007 495 rc->packet_count++;
james@1007 496 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 497 #else
james@1007 498 NET_BUFFER_LIST_FIRST_NB(nbl) = packet;
james@1007 499 #endif
james@1007 500
james@1007 501 if (pi->parse_result == PARSE_OK) {
james@1007 502 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 503 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
james@1007 504 packet, TcpIpChecksumPacketInfo);
james@1023 505 XN_ASSERT(csum_info->Value == 0);
james@1007 506 if (pi->csum_blank || pi->data_validated) {
james@1007 507 BOOLEAN checksum_offload = FALSE;
james@1007 508 /* we know this is IPv4, and we know Linux always validates the IPv4 checksum for us */
james@1007 509 if (xi->setting_csum.V4Receive.IpChecksum) {
james@1007 510 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported) {
james@1007 511 if (XenNet_CheckIpHeaderSum(pi->header, pi->ip4_header_length))
james@1007 512 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
james@1007 513 else
james@1007 514 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
james@1007 515 }
james@1007 516 }
james@1007 517 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6) {
james@1007 518 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported) {
james@1007 519 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
james@1007 520 checksum_offload = TRUE;
james@1007 521 }
james@1007 522 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17) {
james@1007 523 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
james@1007 524 checksum_offload = TRUE;
james@1007 525 }
james@1007 526 if (pi->csum_blank && (!xi->config_csum_rx_dont_fix || !checksum_offload)) {
james@1007 527 XenNet_SumPacketData(pi, packet, TRUE);
james@1007 528 }
james@1007 529 } else if (xi->config_csum_rx_check && pi->ip_version == 4) {
james@1007 530 if (xi->setting_csum.V4Receive.IpChecksum) {
james@1007 531 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported) {
james@1007 532 if (XenNet_CheckIpHeaderSum(pi->header, pi->ip4_header_length))
james@1007 533 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
james@1007 534 else
james@1007 535 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
james@1007 536 }
james@1007 537 }
james@1007 538 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6) {
james@1007 539 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported) {
james@1007 540 if (XenNet_SumPacketData(pi, packet, FALSE)) {
james@1007 541 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
james@1007 542 } else {
james@1007 543 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
james@1007 544 }
james@1007 545 }
james@1007 546 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17) {
james@1007 547 if (XenNet_SumPacketData(pi, packet, FALSE)) {
james@1007 548 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
james@1007 549 } else {
james@1007 550 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
james@1007 551 }
james@1007 552 }
james@1007 553 }
james@1007 554 #else
james@1007 555 csum_info.Value = 0;
james@1007 556 if (pi->csum_blank || pi->data_validated || pi->mss) {
james@1007 557 if (pi->ip_proto == 6) {
james@1007 558 csum_info.Receive.IpChecksumSucceeded = TRUE;
james@1007 559 csum_info.Receive.TcpChecksumSucceeded = TRUE;
james@1007 560 } else if (pi->ip_proto == 17) {
james@1007 561 csum_info.Receive.IpChecksumSucceeded = TRUE;
james@1007 562 csum_info.Receive.UdpChecksumSucceeded = TRUE;
james@1007 563 }
james@1007 564 }
james@1007 565 NET_BUFFER_LIST_INFO(nbl, TcpIpChecksumNetBufferListInfo) = csum_info.Value;
james@1007 566 #endif
james@1007 567 }
james@1007 568
james@1007 569 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 570 if (!rc->first_packet) {
james@1007 571 rc->first_packet = packet;
james@1007 572 } else {
james@1007 573 PACKET_NEXT_PACKET(rc->last_packet) = packet;
james@1007 574 }
james@1007 575 rc->last_packet = packet;
james@1007 576 rc->packet_count++;
james@1007 577 #else
james@1007 578 if (!rc->first_nbl) {
james@1007 579 rc->first_nbl = nbl;
james@1007 580 } else {
james@1007 581 NET_BUFFER_LIST_NEXT_NBL(rc->last_nbl) = nbl;
james@1007 582 }
james@1007 583 rc->last_nbl = nbl;
james@1007 584 NET_BUFFER_LIST_NEXT_NBL(nbl) = NULL;
james@1007 585 rc->nbl_count++;
james@1007 586 if (pi->is_multicast) {
james@1007 587 /* multicast */
james@1007 588 xi->stats.ifHCInMulticastPkts++;
james@1007 589 xi->stats.ifHCInMulticastOctets += NET_BUFFER_DATA_LENGTH(packet);
james@1007 590 } else if (pi->is_broadcast) {
james@1007 591 /* broadcast */
james@1007 592 xi->stats.ifHCInBroadcastPkts++;
james@1011 593 xi->stats.ifHCInBroadcastOctets += NET_BUFFER_DATA_LENGTH(packet);
james@1007 594 } else {
james@1007 595 /* unicast */
james@1007 596 xi->stats.ifHCInUcastPkts++;
james@1011 597 xi->stats.ifHCInUcastOctets += NET_BUFFER_DATA_LENGTH(packet);
james@1007 598 }
james@1007 599 #endif
james@1007 600
james@1007 601 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 602 /* windows gets lazy about ack packets and holds on to them forever under high load situations. we don't like this */
james@1007 603 NdisQueryPacketLength(packet, &packet_length);
james@1007 604 if (pi->ip_proto == 6 && packet_length <= NDIS_STATUS_RESOURCES_MAX_LENGTH)
james@1007 605 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
james@1007 606 else
james@1007 607 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
james@1007 608 #endif
james@1007 609 //FUNCTION_EXIT();
james@1007 610
james@1007 611 InterlockedIncrement(&xi->rx_outstanding);
james@1007 612 //FUNCTION_EXIT();
james@1007 613 return TRUE;
james@1007 614 }
james@1007 615
james@1007 616 static VOID
james@1007 617 XenNet_MakePackets(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi)
james@1007 618 {
james@1007 619 UCHAR psh;
james@1007 620 //PNDIS_BUFFER buffer;
james@1007 621 shared_buffer_t *page_buf;
james@1007 622
james@1007 623 //FUNCTION_ENTER();
james@1007 624
james@1007 625 XenNet_ParsePacketHeader(pi, NULL, 0);
james@1007 626 //pi->split_required = FALSE;
james@1007 627
james@1007 628 if (!XenNet_FilterAcceptPacket(xi, pi)) {
james@1007 629 goto done;
james@1007 630 }
james@1007 631
james@1007 632 if (pi->split_required) {
james@1007 633 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 634 /* need to split to mss for NDIS5 */
james@1007 635 #else
james@1007 636 switch (xi->current_gso_rx_split_type) {
james@1007 637 case RX_LSO_SPLIT_HALF:
james@1007 638 pi->mss = max((pi->tcp_length + 1) / 2, pi->mss);
james@1007 639 break;
james@1007 640 case RX_LSO_SPLIT_NONE:
james@1007 641 pi->mss = 65535;
james@1007 642 break;
james@1007 643 }
james@1007 644 #endif
james@1007 645 }
james@1007 646
james@1007 647 switch (pi->ip_proto) {
james@1007 648 case 6: // TCP
james@1007 649 if (pi->split_required)
james@1007 650 break;
james@1007 651 /* fall through */
james@1007 652 case 17: // UDP
james@1007 653 if (!XenNet_MakePacket(xi, rc, pi)) {
james@1007 654 FUNCTION_MSG("Failed to make packet\n");
james@1007 655 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 656 xi->stat_rx_no_buffer++;
james@1007 657 #else
james@1007 658 xi->stats.ifInDiscards++;
james@1007 659 #endif
james@1007 660 goto done;
james@1007 661 }
james@1007 662 goto done;
james@1007 663 default:
james@1007 664 if (!XenNet_MakePacket(xi, rc, pi)) {
james@1007 665 FUNCTION_MSG("Failed to make packet\n");
james@1007 666 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 667 xi->stat_rx_no_buffer++;
james@1007 668 #else
james@1007 669 xi->stats.ifInDiscards++;
james@1007 670 #endif
james@1007 671 goto done;
james@1007 672 }
james@1007 673 goto done;
james@1007 674 }
james@1007 675
james@1007 676 /* this is the split_required code */
james@1007 677 pi->tcp_remaining = pi->tcp_length;
james@1007 678
james@1007 679 /* we can make certain assumptions here as the following code is only for tcp4 */
james@1007 680 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
james@1007 681 while (pi->tcp_remaining) {
james@1007 682 if (!XenNet_MakePacket(xi, rc, pi)) {
james@1007 683 FUNCTION_MSG("Failed to make packet\n");
james@1007 684 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 685 xi->stat_rx_no_buffer++;
james@1007 686 #else
james@1007 687 xi->stats.ifInDiscards++;
james@1007 688 #endif
james@1007 689 break; /* we are out of memory - just drop the packets */
james@1007 690 }
james@1007 691 if (psh) {
james@1007 692 if (pi->tcp_remaining)
james@1007 693 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
james@1007 694 else
james@1007 695 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
james@1007 696 }
james@1007 697 //XenNet_SumPacketData(pi, packet, TRUE);
james@1007 698 //entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
james@1007 699 //InsertTailList(rx_packet_list, entry);
james@1007 700 }
james@1007 701 done:
james@1007 702 page_buf = pi->first_pb;
james@1007 703 while (page_buf) {
james@1007 704 shared_buffer_t *next_pb = page_buf->next;
james@1007 705 put_pb_on_freelist(xi, page_buf); /* this doesn't actually free the page_puf if there are outstanding references */
james@1007 706 page_buf = next_pb;
james@1007 707 }
james@1007 708 XenNet_ClearPacketInfo(pi);
james@1007 709 //FUNCTION_EXIT();
james@1007 710 return;
james@1007 711 }
james@1007 712
james@1007 713 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 714 /* called at DISPATCH_LEVEL */
james@1007 715 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
james@1007 716 VOID
james@1007 717 XenNet_ReturnPacket(NDIS_HANDLE adapter_context, PNDIS_PACKET packet) {
james@1007 718 struct xennet_info *xi = adapter_context;
james@1007 719 PNDIS_BUFFER buffer;
james@1007 720 shared_buffer_t *page_buf = PACKET_FIRST_PB(packet);
james@1007 721
james@1007 722 //FUNCTION_ENTER();
james@1007 723 NdisUnchainBufferAtFront(packet, &buffer);
james@1007 724
james@1007 725 while (buffer) {
james@1007 726 shared_buffer_t *next_buf;
james@1023 727 XN_ASSERT(page_buf);
james@1007 728 next_buf = page_buf->next;
james@1007 729 if (!page_buf->virtual) {
james@1007 730 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
james@1007 731 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(buffer) - 1);
james@1007 732 } else {
james@1007 733 if (buffer != page_buf->mdl)
james@1007 734 NdisFreeBuffer(buffer);
james@1007 735 put_pb_on_freelist(xi, page_buf);
james@1007 736 }
james@1007 737 NdisUnchainBufferAtFront(packet, &buffer);
james@1007 738 page_buf = next_buf;
james@1007 739 }
james@1007 740
james@1007 741 NdisFreePacket(packet);
james@1017 742 InterlockedDecrement(&xi->rx_outstanding);
james@1007 743 if (!xi->rx_outstanding && xi->device_state != DEVICE_STATE_ACTIVE)
james@1007 744 KeSetEvent(&xi->rx_idle_event, IO_NO_INCREMENT, FALSE);
james@1007 745 //FUNCTION_EXIT();
james@1007 746 }
james@1007 747 #else
james@1007 748 /* called at <= DISPATCH_LEVEL */
james@1007 749 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
james@1007 750 VOID
james@1007 751 XenNet_ReturnNetBufferLists(NDIS_HANDLE adapter_context, PNET_BUFFER_LIST curr_nbl, ULONG return_flags)
james@1007 752 {
james@1007 753 struct xennet_info *xi = adapter_context;
james@1007 754 UNREFERENCED_PARAMETER(return_flags);
james@1007 755
james@1007 756 //FUNCTION_ENTER();
james@1007 757
james@1007 758 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
james@1007 759
james@1023 760 XN_ASSERT(xi);
james@1007 761 while (curr_nbl)
james@1007 762 {
james@1007 763 PNET_BUFFER_LIST next_nbl;
james@1007 764 PNET_BUFFER curr_nb;
james@1007 765
james@1007 766 next_nbl = NET_BUFFER_LIST_NEXT_NBL(curr_nbl);
james@1007 767 curr_nb = NET_BUFFER_LIST_FIRST_NB(curr_nbl);
james@1007 768 while (curr_nb)
james@1007 769 {
james@1007 770 PNET_BUFFER next_nb;
james@1007 771 PMDL curr_mdl;
james@1007 772 shared_buffer_t *page_buf;
james@1007 773
james@1007 774 next_nb = NET_BUFFER_NEXT_NB(curr_nb);
james@1007 775 curr_mdl = NET_BUFFER_FIRST_MDL(curr_nb);
james@1007 776 page_buf = NB_FIRST_PB(curr_nb);
james@1007 777 while (curr_mdl)
james@1007 778 {
james@1007 779 shared_buffer_t *next_buf;
james@1007 780 PMDL next_mdl;
james@1007 781
james@1023 782 XN_ASSERT(page_buf); /* make sure that there is a pb to match this mdl */
james@1007 783 next_mdl = curr_mdl->Next;
james@1007 784 next_buf = page_buf->next;
james@1007 785 if (!page_buf->virtual)
james@1007 786 {
james@1007 787 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
james@1007 788 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(curr_mdl) - 1);
james@1007 789 }
james@1007 790 else
james@1007 791 {
james@1007 792 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
james@1007 793 if (curr_mdl != page_buf->mdl)
james@1007 794 {
james@1007 795 //KdPrint((__DRIVER_NAME " curr_mdl = %p, page_buf->mdl = %p\n", curr_mdl, page_buf->mdl));
james@1007 796 IoFreeMdl(curr_mdl);
james@1007 797 }
james@1007 798 put_pb_on_freelist(xi, page_buf);
james@1007 799 }
james@1007 800 curr_mdl = next_mdl;
james@1007 801 page_buf = next_buf;
james@1007 802 }
james@1007 803
james@1007 804 NdisFreeNetBuffer(curr_nb);
james@1007 805 InterlockedDecrement(&xi->rx_outstanding);
james@1007 806
james@1007 807 curr_nb = next_nb;
james@1007 808 }
james@1007 809 NdisFreeNetBufferList(curr_nbl);
james@1007 810 curr_nbl = next_nbl;
james@1007 811 }
james@1007 812
james@1007 813 if (!xi->rx_outstanding && xi->device_state != DEVICE_STATE_ACTIVE)
james@1007 814 KeSetEvent(&xi->rx_idle_event, IO_NO_INCREMENT, FALSE);
james@1007 815
james@1007 816 //FUNCTION_EXIT();
james@1007 817 }
james@1007 818 #endif
james@1007 819
james@1007 820 /* We limit the number of packets per interrupt so that acks get a chance
james@1007 821 under high rx load. The DPC is immediately re-scheduled */
james@1007 822
james@1007 823 #define MAXIMUM_PACKETS_PER_INDICATE 32
james@1007 824
james@1007 825 #define MAXIMUM_PACKETS_PER_INTERRUPT 2560 /* this is calculated before large packet split */
james@1007 826 #define MAXIMUM_DATA_PER_INTERRUPT (MAXIMUM_PACKETS_PER_INTERRUPT * 1500) /* help account for large packets */
james@1007 827
james@1007 828 // Called at DISPATCH_LEVEL
james@1007 829 BOOLEAN
james@1007 830 XenNet_RxBufferCheck(struct xennet_info *xi)
james@1007 831 {
james@1007 832 RING_IDX cons, prod;
james@1007 833 ULONG packet_count = 0;
james@1007 834 ULONG packet_data = 0;
james@1007 835 ULONG buffer_count = 0;
james@1007 836 USHORT id;
james@1007 837 int more_to_do = FALSE;
james@1007 838 shared_buffer_t *page_buf;
james@1007 839 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 840 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
james@1007 841 PNDIS_PACKET first_header_only_packet;
james@1007 842 PNDIS_PACKET last_header_only_packet;
james@1007 843 #else
james@1007 844 #endif
james@1007 845 //ULONG nbl_count = 0;
james@1007 846 ULONG interim_packet_data = 0;
james@1007 847 struct netif_extra_info *ei;
james@1007 848 rx_context_t rc;
james@1007 849 packet_info_t *pi = &xi->rxpi[KeGetCurrentProcessorNumber() & 0xff];
james@1007 850 shared_buffer_t *head_buf = NULL;
james@1007 851 shared_buffer_t *tail_buf = NULL;
james@1007 852 shared_buffer_t *last_buf = NULL;
james@1007 853 BOOLEAN extra_info_flag = FALSE;
james@1007 854 BOOLEAN more_data_flag = FALSE;
james@1007 855 BOOLEAN dont_set_event;
james@1007 856 //FUNCTION_ENTER();
james@1007 857
james@1007 858 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 859 rc.first_packet = NULL;
james@1007 860 rc.last_packet = NULL;
james@1007 861 rc.packet_count = 0;
james@1007 862 #else
james@1007 863 rc.first_nbl = NULL;
james@1007 864 rc.last_nbl = NULL;
james@1007 865 rc.packet_count = 0;
james@1007 866 rc.nbl_count = 0;
james@1007 867 #endif
james@1007 868
james@1007 869 /* get all the buffers off the ring as quickly as possible so the lock is held for a minimum amount of time */
james@1007 870 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
james@1007 871
james@1007 872 if (xi->device_state != DEVICE_STATE_ACTIVE) {
james@1007 873 /* there is a chance that our Dpc had been queued just before the shutdown... */
james@1007 874 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
james@1007 875 return FALSE;
james@1007 876 }
james@1007 877
james@1007 878 if (xi->rx_partial_buf) {
james@1007 879 head_buf = xi->rx_partial_buf;
james@1007 880 tail_buf = xi->rx_partial_buf;
james@1007 881 while (tail_buf->next)
james@1007 882 tail_buf = tail_buf->next;
james@1007 883 more_data_flag = xi->rx_partial_more_data_flag;
james@1007 884 extra_info_flag = xi->rx_partial_extra_info_flag;
james@1007 885 xi->rx_partial_buf = NULL;
james@1007 886 }
james@1007 887
james@1007 888 do {
james@1007 889 prod = xi->rx_ring.sring->rsp_prod;
james@1007 890 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
james@1007 891
james@1007 892 for (cons = xi->rx_ring.rsp_cons; cons != prod && packet_count < MAXIMUM_PACKETS_PER_INTERRUPT && packet_data < MAXIMUM_DATA_PER_INTERRUPT; cons++) {
james@1007 893 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
james@1007 894 page_buf = xi->rx_ring_pbs[id];
james@1023 895 XN_ASSERT(page_buf);
james@1007 896 xi->rx_ring_pbs[id] = NULL;
james@1007 897 xi->rx_id_free++;
james@1007 898 memcpy(&page_buf->rsp, RING_GET_RESPONSE(&xi->rx_ring, cons), max(sizeof(struct netif_rx_response), sizeof(struct netif_extra_info)));
james@1007 899 if (!extra_info_flag) {
james@1007 900 if (page_buf->rsp.status <= 0 || page_buf->rsp.offset + page_buf->rsp.status > PAGE_SIZE) {
james@1007 901 KdPrint((__DRIVER_NAME " Error: rsp offset %d, size %d\n",
james@1007 902 page_buf->rsp.offset, page_buf->rsp.status));
james@1023 903 XN_ASSERT(!extra_info_flag);
james@1007 904 put_pb_on_freelist(xi, page_buf);
james@1007 905 continue;
james@1007 906 }
james@1007 907 }
james@1007 908
james@1007 909 if (!head_buf) {
james@1007 910 head_buf = page_buf;
james@1007 911 tail_buf = page_buf;
james@1007 912 } else {
james@1007 913 tail_buf->next = page_buf;
james@1007 914 tail_buf = page_buf;
james@1007 915 }
james@1007 916 page_buf->next = NULL;
james@1007 917
james@1007 918 if (extra_info_flag) {
james@1007 919 ei = (struct netif_extra_info *)&page_buf->rsp;
james@1007 920 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
james@1007 921 } else {
james@1007 922 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
james@1007 923 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
james@1007 924 interim_packet_data += page_buf->rsp.status;
james@1007 925 }
james@1007 926
james@1007 927 if (!extra_info_flag && !more_data_flag) {
james@1007 928 last_buf = page_buf;
james@1007 929 packet_count++;
james@1007 930 packet_data += interim_packet_data;
james@1007 931 interim_packet_data = 0;
james@1007 932 }
james@1007 933 buffer_count++;
james@1007 934 }
james@1007 935 xi->rx_ring.rsp_cons = cons;
james@1007 936
james@1007 937 /* Give netback more buffers */
james@1007 938 XenNet_FillRing(xi);
james@1007 939
james@1007 940 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
james@1007 941 break;
james@1007 942
james@1007 943 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx_ring);
james@1007 944 if (!more_to_do) {
james@1007 945 xi->rx_ring.sring->rsp_event = xi->rx_ring.rsp_cons + 1;
james@1007 946 KeMemoryBarrier();
james@1007 947 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx_ring);
james@1007 948 }
james@1007 949 } while (more_to_do);
james@1007 950
james@1007 951 /* anything past last_buf belongs to an incomplete packet... */
james@1007 952 if (last_buf && last_buf->next)
james@1007 953 {
james@1007 954 KdPrint((__DRIVER_NAME " Partial receive\n"));
james@1007 955 xi->rx_partial_buf = last_buf->next;
james@1007 956 xi->rx_partial_more_data_flag = more_data_flag;
james@1007 957 xi->rx_partial_extra_info_flag = extra_info_flag;
james@1007 958 last_buf->next = NULL;
james@1007 959 }
james@1007 960
james@1007 961 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
james@1007 962
james@1007 963 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
james@1007 964 {
james@1007 965 /* fire again immediately */
james@1007 966 KdPrint((__DRIVER_NAME " Dpc Duration Exceeded\n"));
james@1007 967 /* we want the Dpc on the end of the queue. By definition we are already on the right CPU so we know the Dpc queue will be run immediately */
james@1007 968 // KeSetImportanceDpc(&xi->rxtx_dpc, MediumImportance);
james@1007 969 KeInsertQueueDpc(&xi->rxtx_dpc, NULL, NULL);
james@1007 970 /* dont set an event in TX path */
james@1007 971 dont_set_event = TRUE;
james@1007 972 }
james@1007 973 else
james@1007 974 {
james@1007 975 /* make sure the Dpc queue is run immediately next interrupt */
james@1007 976 // KeSetImportanceDpc(&xi->rxtx_dpc, HighImportance);
james@1007 977 /* set an event in TX path */
james@1007 978 dont_set_event = FALSE;
james@1007 979 }
james@1007 980
james@1007 981 /* make packets out of the buffers */
james@1007 982 page_buf = head_buf;
james@1007 983 extra_info_flag = FALSE;
james@1007 984 more_data_flag = FALSE;
james@1007 985
james@1007 986 while (page_buf) {
james@1007 987 shared_buffer_t *next_buf = page_buf->next;
james@1007 988 PMDL mdl;
james@1007 989
james@1007 990 page_buf->next = NULL;
james@1007 991 if (extra_info_flag) {
james@1007 992 //KdPrint((__DRIVER_NAME " processing extra info\n"));
james@1007 993 ei = (struct netif_extra_info *)&page_buf->rsp;
james@1007 994 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
james@1007 995 switch (ei->type)
james@1007 996 {
james@1007 997 case XEN_NETIF_EXTRA_TYPE_GSO:
james@1007 998 switch (ei->u.gso.type) {
james@1007 999 case XEN_NETIF_GSO_TYPE_TCPV4:
james@1007 1000 pi->mss = ei->u.gso.size;
james@1023 1001 // TODO - put this assertion somewhere XN_ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
james@1007 1002 break;
james@1007 1003 default:
james@1007 1004 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
james@1007 1005 break;
james@1007 1006 }
james@1007 1007 break;
james@1007 1008 default:
james@1007 1009 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
james@1007 1010 break;
james@1007 1011 }
james@1007 1012 put_pb_on_freelist(xi, page_buf);
james@1007 1013 } else {
james@1023 1014 XN_ASSERT(!page_buf->rsp.offset);
james@1007 1015 if (!more_data_flag) { // handling the packet's 1st buffer
james@1007 1016 if (page_buf->rsp.flags & NETRXF_csum_blank)
james@1007 1017 pi->csum_blank = TRUE;
james@1007 1018 if (page_buf->rsp.flags & NETRXF_data_validated)
james@1007 1019 pi->data_validated = TRUE;
james@1007 1020 }
james@1007 1021 mdl = page_buf->mdl;
james@1007 1022 mdl->ByteCount = page_buf->rsp.status; //NdisAdjustBufferLength(mdl, page_buf->rsp.status);
james@1007 1023 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
james@1007 1024 if (pi->first_pb) {
james@1023 1025 XN_ASSERT(pi->curr_pb);
james@1007 1026 //KdPrint((__DRIVER_NAME " additional buffer\n"));
james@1007 1027 pi->curr_pb->next = page_buf;
james@1007 1028 pi->curr_pb = page_buf;
james@1023 1029 XN_ASSERT(pi->curr_mdl);
james@1007 1030 pi->curr_mdl->Next = mdl;
james@1007 1031 pi->curr_mdl = mdl;
james@1007 1032 } else {
james@1007 1033 pi->first_pb = page_buf;
james@1007 1034 pi->curr_pb = page_buf;
james@1007 1035 pi->first_mdl = mdl;
james@1007 1036 pi->curr_mdl = mdl;
james@1007 1037 }
james@1007 1038 //pi->mdl_count++;
james@1007 1039 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
james@1007 1040 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
james@1007 1041 pi->total_length = pi->total_length + page_buf->rsp.status;
james@1007 1042 }
james@1007 1043
james@1007 1044 /* Packet done, add it to the list */
james@1007 1045 if (!more_data_flag && !extra_info_flag) {
james@1007 1046 pi->curr_pb = pi->first_pb;
james@1007 1047 pi->curr_mdl = pi->first_mdl;
james@1007 1048 XenNet_MakePackets(xi, &rc, pi);
james@1007 1049 }
james@1007 1050
james@1007 1051 page_buf = next_buf;
james@1007 1052 }
james@1023 1053 XN_ASSERT(!more_data_flag && !extra_info_flag);
james@1007 1054
james@1007 1055 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 1056 packet_count = 0;
james@1007 1057 first_header_only_packet = NULL;
james@1007 1058 last_header_only_packet = NULL;
james@1007 1059
james@1007 1060 while (rc.first_packet) {
james@1007 1061 PNDIS_PACKET packet;
james@1007 1062 NDIS_STATUS status;
james@1007 1063
james@1007 1064 packet = rc.first_packet;
james@1023 1065 XN_ASSERT(PACKET_FIRST_PB(packet));
james@1007 1066 rc.first_packet = PACKET_NEXT_PACKET(packet);
james@1007 1067 status = NDIS_GET_PACKET_STATUS(packet);
james@1007 1068 if (status == NDIS_STATUS_RESOURCES) {
james@1007 1069 if (!first_header_only_packet) {
james@1007 1070 first_header_only_packet = packet;
james@1007 1071 } else {
james@1007 1072 PACKET_NEXT_PACKET(last_header_only_packet) = packet;
james@1007 1073 }
james@1007 1074 last_header_only_packet = packet;
james@1007 1075 PACKET_NEXT_PACKET(packet) = NULL;
james@1007 1076 }
james@1007 1077 packets[packet_count++] = packet;
james@1007 1078 /* if we indicate a packet with NDIS_STATUS_RESOURCES then any following packet can't be NDIS_STATUS_SUCCESS */
james@1007 1079 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || !rc.first_packet
james@1007 1080 || (NDIS_GET_PACKET_STATUS(rc.first_packet) == NDIS_STATUS_SUCCESS
james@1007 1081 && status == NDIS_STATUS_RESOURCES)) {
james@1007 1082 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
james@1007 1083 packet_count = 0;
james@1007 1084 }
james@1007 1085 }
james@1007 1086 /* now return the packets for which we indicated NDIS_STATUS_RESOURCES */
james@1007 1087 while (first_header_only_packet) {
james@1007 1088 PNDIS_PACKET packet = first_header_only_packet;
james@1007 1089 first_header_only_packet = PACKET_NEXT_PACKET(packet);
james@1007 1090 XenNet_ReturnPacket(xi, packet);
james@1007 1091 }
james@1007 1092 #else
james@1007 1093 if (rc.first_nbl) {
james@1007 1094 NdisMIndicateReceiveNetBufferLists(xi->adapter_handle, rc.first_nbl,
james@1007 1095 NDIS_DEFAULT_PORT_NUMBER, rc.nbl_count,
james@1007 1096 NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL
james@1007 1097 //| NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE
james@1007 1098 | NDIS_RECEIVE_FLAGS_PERFECT_FILTERED);
james@1007 1099 }
james@1007 1100 #endif
james@1007 1101 //FUNCTION_EXIT();
james@1007 1102 return dont_set_event;
james@1007 1103 }
james@1007 1104
james@1007 1105 static VOID
james@1007 1106 XenNet_BufferFree(xennet_info_t *xi)
james@1007 1107 {
james@1007 1108 shared_buffer_t *sb;
james@1007 1109 int i;
james@1007 1110
james@1007 1111 for (i = 0; i < NET_RX_RING_SIZE; i++) {
james@1007 1112 if (xi->rx_ring_pbs[i] != NULL) {
james@1007 1113 put_pb_on_freelist(xi, xi->rx_ring_pbs[i]);
james@1007 1114 xi->rx_ring_pbs[i] = NULL;
james@1007 1115 }
james@1007 1116 }
james@1007 1117
james@1007 1118 /* because we are shutting down this won't allocate new ones */
james@1007 1119 while ((sb = get_pb_from_freelist(xi)) != NULL) {
james@1007 1120 XnEndAccess(xi->handle,
james@1007 1121 sb->gref, FALSE, (ULONG)'XNRX');
james@1007 1122 IoFreeMdl(sb->mdl);
james@1007 1123 ExFreePoolWithTag(sb->virtual, XENNET_POOL_TAG);
james@1007 1124 ExFreePoolWithTag(sb, XENNET_POOL_TAG);
james@1007 1125 }
james@1007 1126 while ((sb = get_hb_from_freelist(xi)) != NULL) {
james@1007 1127 IoFreeMdl(sb->mdl);
james@1007 1128 ExFreePoolWithTag(sb, XENNET_POOL_TAG);
james@1007 1129 }
james@1007 1130 }
james@1007 1131
james@1007 1132 BOOLEAN
james@1007 1133 XenNet_RxInit(xennet_info_t *xi) {
james@1007 1134 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 1135 NDIS_STATUS status;
james@1007 1136 #else
james@1007 1137 NET_BUFFER_LIST_POOL_PARAMETERS nbl_pool_parameters;
james@1007 1138 NET_BUFFER_POOL_PARAMETERS nb_pool_parameters;
james@1007 1139 #endif
james@1007 1140 int ret;
james@1007 1141 int i;
james@1007 1142
james@1007 1143 FUNCTION_ENTER();
james@1007 1144
james@1007 1145 // this stuff needs to be done once only...
james@1007 1146 KeInitializeSpinLock(&xi->rx_lock);
james@1007 1147 KeInitializeEvent(&xi->rx_idle_event, SynchronizationEvent, FALSE);
james@1007 1148 xi->rxpi = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(packet_info_t) * NdisSystemProcessorCount(), XENNET_POOL_TAG, NormalPoolPriority);
james@1007 1149 if (!xi->rxpi) {
james@1007 1150 KdPrint(("ExAllocatePoolWithTagPriority failed\n"));
james@1007 1151 return FALSE;
james@1007 1152 }
james@1007 1153 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount());
james@1007 1154
james@1007 1155 ret = stack_new(&xi->rx_pb_stack, NET_RX_RING_SIZE * 4);
james@1007 1156 if (!ret) {
james@1007 1157 FUNCTION_MSG("Failed to allocate rx_pb_stack\n");
james@1007 1158 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
james@1007 1159 return FALSE;
james@1007 1160 }
james@1007 1161 ret = stack_new(&xi->rx_hb_stack, NET_RX_RING_SIZE * 4);
james@1007 1162 if (!ret) {
james@1007 1163 FUNCTION_MSG("Failed to allocate rx_hb_stack\n");
james@1007 1164 stack_delete(xi->rx_pb_stack, NULL, NULL);
james@1007 1165 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
james@1007 1166 return FALSE;
james@1007 1167 }
james@1007 1168
james@1007 1169 xi->rx_id_free = NET_RX_RING_SIZE;
james@1007 1170 xi->rx_outstanding = 0;
james@1007 1171
james@1007 1172 for (i = 0; i < NET_RX_RING_SIZE; i++) {
james@1007 1173 xi->rx_ring_pbs[i] = NULL;
james@1007 1174 }
james@1007 1175
james@1007 1176 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 1177 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4, PROTOCOL_RESERVED_SIZE_IN_PACKET);
james@1007 1178 if (status != NDIS_STATUS_SUCCESS) {
james@1007 1179 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
james@1007 1180 return FALSE;
james@1007 1181 }
james@1007 1182 #else
james@1007 1183 nbl_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
james@1007 1184 nbl_pool_parameters.Header.Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
james@1007 1185 nbl_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
james@1007 1186 nbl_pool_parameters.ProtocolId = NDIS_PROTOCOL_ID_DEFAULT;
james@1007 1187 nbl_pool_parameters.fAllocateNetBuffer = FALSE;
james@1007 1188 nbl_pool_parameters.ContextSize = 0;
james@1007 1189 nbl_pool_parameters.PoolTag = XENNET_POOL_TAG;
james@1007 1190 nbl_pool_parameters.DataSize = 0; /* NET_BUFFERS are always allocated separately */
james@1007 1191
james@1007 1192 xi->rx_nbl_pool = NdisAllocateNetBufferListPool(xi->adapter_handle, &nbl_pool_parameters);
james@1007 1193 if (!xi->rx_nbl_pool) {
james@1007 1194 KdPrint(("NdisAllocateNetBufferListPool failed\n"));
james@1007 1195 return FALSE;
james@1007 1196 }
james@1007 1197
james@1007 1198 nb_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
james@1007 1199 nb_pool_parameters.Header.Revision = NET_BUFFER_POOL_PARAMETERS_REVISION_1;
james@1007 1200 nb_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_POOL_PARAMETERS_REVISION_1;
james@1007 1201 nb_pool_parameters.PoolTag = XENNET_POOL_TAG;
james@1007 1202 nb_pool_parameters.DataSize = 0; /* the buffers come from the ring */
james@1011 1203 xi->rx_packet_pool = NdisAllocateNetBufferPool(xi->adapter_handle, &nb_pool_parameters);
james@1011 1204 if (!xi->rx_packet_pool) {
james@1011 1205 KdPrint(("NdisAllocateNetBufferPool (rx_packet_pool) failed\n"));
james@1007 1206 return FALSE;
james@1007 1207 }
james@1007 1208 #endif
james@1007 1209 XenNet_FillRing(xi);
james@1007 1210
james@1007 1211 FUNCTION_EXIT();
james@1007 1212
james@1007 1213 return TRUE;
james@1007 1214 }
james@1007 1215
james@1007 1216 VOID
james@1007 1217 XenNet_RxShutdown(xennet_info_t *xi) {
james@1007 1218 KIRQL old_irql;
james@1017 1219 UNREFERENCED_PARAMETER(xi);
james@1007 1220
james@1007 1221 FUNCTION_ENTER();
james@1007 1222
james@1007 1223 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
james@1007 1224 while (xi->rx_outstanding) {
james@1017 1225 FUNCTION_MSG("Waiting for %d packets to be returned\n", xi->rx_outstanding);
james@1007 1226 KeReleaseSpinLock(&xi->rx_lock, old_irql);
james@1007 1227 KeWaitForSingleObject(&xi->rx_idle_event, Executive, KernelMode, FALSE, NULL);
james@1007 1228 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
james@1007 1229 }
james@1007 1230 KeReleaseSpinLock(&xi->rx_lock, old_irql);
james@1007 1231
james@1007 1232 XenNet_BufferFree(xi);
james@1007 1233
james@1007 1234 stack_delete(xi->rx_pb_stack, NULL, NULL);
james@1007 1235 stack_delete(xi->rx_hb_stack, NULL, NULL);
james@1007 1236
james@1007 1237
james@1007 1238 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
james@1007 1239
james@1007 1240 #if NTDDI_VERSION < NTDDI_VISTA
james@1007 1241 NdisFreePacketPool(xi->rx_packet_pool);
james@1007 1242 #else
james@1011 1243 NdisFreeNetBufferPool(xi->rx_packet_pool);
james@1007 1244 NdisFreeNetBufferListPool(xi->rx_nbl_pool);
james@1007 1245 #endif
james@1007 1246
james@1007 1247 FUNCTION_EXIT();
james@1007 1248 return;
james@1007 1249 }