win-pvdrivers

view xennet/xennet_rx.c @ 1106:2d392ecdd366

Fix race is xenvbd causing 30 second freeze under high load
author James Harper <james.harper@bendigoit.com.au>
date Tue Nov 11 23:08:11 2014 +1100 (2014-11-11)
parents ef445b2286f7
children
line source
1 /*
2 PV Drivers for Windows Xen HVM Domains
4 Copyright (c) 2014, James Harper
5 All rights reserved.
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9 * Redistributions of source code must retain the above copyright
10 notice, this list of conditions and the following disclaimer.
11 * Redistributions in binary form must reproduce the above copyright
12 notice, this list of conditions and the following disclaimer in the
13 documentation and/or other materials provided with the distribution.
14 * Neither the name of James Harper nor the
15 names of its contributors may be used to endorse or promote products
16 derived from this software without specific prior written permission.
18 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 DISCLAIMED. IN NO EVENT SHALL JAMES HARPER BE LIABLE FOR ANY
22 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
30 #include "xennet.h"
32 static __inline shared_buffer_t *
33 get_pb_from_freelist(struct xennet_info *xi) {
34 shared_buffer_t *pb;
35 PVOID ptr_ref;
37 if (stack_pop(xi->rx_pb_stack, &ptr_ref)) {
38 pb = ptr_ref;
39 pb->ref_count = 1;
40 InterlockedDecrement(&xi->rx_pb_free);
41 return pb;
42 }
44 /* don't allocate a new one if we are shutting down */
45 if (xi->device_state != DEVICE_STATE_INITIALISING && xi->device_state != DEVICE_STATE_ACTIVE)
46 return NULL;
48 pb = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(shared_buffer_t), XENNET_POOL_TAG, LowPoolPriority);
49 if (!pb)
50 return NULL;
51 pb->virtual = ExAllocatePoolWithTagPriority(NonPagedPool, PAGE_SIZE, XENNET_POOL_TAG, LowPoolPriority);
52 if (!pb->virtual) {
53 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
54 return NULL;
55 }
56 pb->mdl = IoAllocateMdl(pb->virtual, PAGE_SIZE, FALSE, FALSE, NULL);
57 if (!pb->mdl) {
58 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
59 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
60 return NULL;
61 }
62 pb->gref = (grant_ref_t)XnGrantAccess(xi->handle,
63 (ULONG)(MmGetPhysicalAddress(pb->virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
64 if (pb->gref == INVALID_GRANT_REF) {
65 IoFreeMdl(pb->mdl);
66 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
67 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
68 return NULL;
69 }
70 MmBuildMdlForNonPagedPool(pb->mdl);
71 pb->ref_count = 1;
72 return pb;
73 }
75 static __inline VOID
76 ref_pb(struct xennet_info *xi, shared_buffer_t *pb) {
77 UNREFERENCED_PARAMETER(xi);
78 InterlockedIncrement(&pb->ref_count);
79 }
81 static __inline VOID
82 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb) {
83 int ref = InterlockedDecrement(&pb->ref_count);
84 XN_ASSERT(ref >= 0);
85 if (ref == 0) {
86 //NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
87 //NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
88 if (xi->rx_pb_free > RX_MAX_PB_FREELIST) {
89 XnEndAccess(xi->handle, pb->gref, FALSE, (ULONG)'XNRX');
90 IoFreeMdl(pb->mdl);
91 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
92 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
93 return;
94 }
95 pb->mdl->ByteCount = PAGE_SIZE;
96 pb->mdl->Next = NULL;
97 pb->next = NULL;
98 stack_push(xi->rx_pb_stack, pb);
99 InterlockedIncrement(&xi->rx_pb_free);
100 }
101 }
103 static __inline shared_buffer_t *
104 get_hb_from_freelist(struct xennet_info *xi) {
105 shared_buffer_t *hb;
106 PVOID ptr_ref;
108 if (stack_pop(xi->rx_hb_stack, &ptr_ref)) {
109 hb = ptr_ref;
110 InterlockedDecrement(&xi->rx_hb_free);
111 return hb;
112 }
114 /* don't allocate a new one if we are shutting down */
115 if (xi->device_state != DEVICE_STATE_INITIALISING && xi->device_state != DEVICE_STATE_ACTIVE)
116 return NULL;
118 hb = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, XENNET_POOL_TAG, LowPoolPriority);
119 if (!hb)
120 return NULL;
121 NdisZeroMemory(hb, sizeof(shared_buffer_t));
122 hb->mdl = IoAllocateMdl(hb + 1, MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, FALSE, FALSE, NULL);
123 if (!hb->mdl) {
124 ExFreePoolWithTag(hb, XENNET_POOL_TAG);
125 return NULL;
126 }
127 MmBuildMdlForNonPagedPool(hb->mdl);
128 return hb;
129 }
131 static __inline VOID
132 put_hb_on_freelist(struct xennet_info *xi, shared_buffer_t *hb) {
133 XN_ASSERT(xi);
134 hb->mdl->ByteCount = sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH;
135 hb->mdl->Next = NULL;
136 hb->next = NULL;
137 stack_push(xi->rx_hb_stack, hb);
138 InterlockedIncrement(&xi->rx_hb_free);
139 }
141 // Called at DISPATCH_LEVEL with rx lock held
142 static VOID
143 XenNet_FillRing(struct xennet_info *xi) {
144 unsigned short id;
145 shared_buffer_t *page_buf;
146 ULONG i, notify;
147 ULONG batch_target;
148 RING_IDX req_prod = xi->rx_ring.req_prod_pvt;
149 netif_rx_request_t *req;
151 //FUNCTION_ENTER();
153 if (xi->device_state != DEVICE_STATE_ACTIVE)
154 return;
156 batch_target = xi->rx_target - (req_prod - xi->rx_ring.rsp_cons);
158 if (batch_target < (xi->rx_target >> 2)) {
159 //FUNCTION_EXIT();
160 return; /* only refill if we are less than 3/4 full already */
161 }
163 for (i = 0; i < batch_target; i++) {
164 page_buf = get_pb_from_freelist(xi);
165 if (!page_buf) {
166 FUNCTION_MSG("Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target);
167 break;
168 }
169 xi->rx_id_free--;
171 /* Give to netback */
172 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
173 XN_ASSERT(xi->rx_ring_pbs[id] == NULL);
174 xi->rx_ring_pbs[id] = page_buf;
175 req = RING_GET_REQUEST(&xi->rx_ring, req_prod + i);
176 req->id = id;
177 req->gref = page_buf->gref;
178 XN_ASSERT(req->gref != INVALID_GRANT_REF);
179 }
180 KeMemoryBarrier();
181 xi->rx_ring.req_prod_pvt = req_prod + i;
182 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx_ring, notify);
183 if (notify) {
184 XnNotify(xi->handle, xi->event_channel);
185 }
187 //FUNCTION_EXIT();
189 return;
190 }
192 #if NTDDI_VERSION < NTDDI_VISTA
193 typedef struct {
194 PNDIS_PACKET first_packet;
195 PNDIS_PACKET last_packet;
196 ULONG packet_count;
197 } rx_context_t;
198 #else
199 typedef struct {
200 PNET_BUFFER_LIST first_nbl;
201 PNET_BUFFER_LIST last_nbl;
202 ULONG packet_count;
203 ULONG nbl_count;
204 } rx_context_t;
205 #endif
207 #if NTDDI_VERSION < NTDDI_VISTA
208 /*
209 NDIS5 appears to insist that the checksum on received packets is correct, and won't
210 believe us when we lie about it, which happens when the packet is generated on the
211 same bridge in Dom0. Doh!
212 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
213 */
215 static BOOLEAN
216 XenNet_SumPacketData(
217 packet_info_t *pi,
218 PNDIS_PACKET packet,
219 BOOLEAN set_csum) {
220 USHORT i;
221 PUCHAR buffer;
222 PMDL mdl;
223 UINT total_length;
224 UINT data_length;
225 UINT buffer_length;
226 USHORT buffer_offset;
227 ULONG csum;
228 PUSHORT csum_ptr;
229 USHORT remaining;
230 USHORT ip4_length;
231 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
233 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
234 if (!buffer) {
235 FUNCTION_MSG("NdisGetFirstBufferFromPacketSafe failed, buffer == NULL\n");
236 return FALSE;
237 }
238 XN_ASSERT(mdl);
240 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
241 data_length = ip4_length + XN_HDR_SIZE;
243 if ((USHORT)data_length > total_length) {
244 FUNCTION_MSG("Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length);
245 return FALSE;
246 }
248 switch (pi->ip_proto) {
249 case 6:
250 XN_ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
251 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
252 break;
253 case 17:
254 XN_ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
255 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
256 break;
257 default:
258 FUNCTION_MSG("Don't know how to calc sum for IP Proto %d\n", pi->ip_proto);
259 //FUNCTION_EXIT();
260 return FALSE; // should never happen
261 }
263 if (set_csum)
264 *csum_ptr = 0;
266 csum = 0;
267 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
268 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
269 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
271 remaining = ip4_length - pi->ip4_header_length;
273 csum += remaining;
275 csum_span = FALSE;
276 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
277 while (i < data_length) {
278 /* don't include the checksum field itself in the calculation */
279 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6)) {
280 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
281 i += 2;
282 buffer_offset += 2;
283 continue;
284 }
285 if (csum_span) {
286 /* the other half of the next bit */
287 XN_ASSERT(buffer_offset == 0);
288 csum += (USHORT)buffer[buffer_offset];
289 csum_span = FALSE;
290 i += 1;
291 buffer_offset += 1;
292 } else if (buffer_offset == buffer_length - 1) {
293 /* deal with a buffer ending on an odd byte boundary */
294 csum += (USHORT)buffer[buffer_offset] << 8;
295 csum_span = TRUE;
296 i += 1;
297 buffer_offset += 1;
298 } else {
299 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
300 i += 2;
301 buffer_offset += 2;
302 }
303 if (buffer_offset == buffer_length && i < total_length) {
304 NdisGetNextBuffer(mdl, &mdl);
305 if (mdl == NULL) {
306 FUNCTION_MSG(__DRIVER_NAME " Ran out of buffers\n");
307 return FALSE; // should never happen
308 }
309 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
310 XN_ASSERT(buffer_length);
311 buffer_offset = 0;
312 }
313 }
315 while (csum & 0xFFFF0000)
316 csum = (csum & 0xFFFF) + (csum >> 16);
318 if (set_csum) {
319 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
320 } else {
321 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
322 }
323 return TRUE;
324 }
325 #endif
327 static BOOLEAN
328 XenNet_MakePacket(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi) {
329 #if NTDDI_VERSION < NTDDI_VISTA
330 NDIS_STATUS status;
331 PNDIS_PACKET packet;
332 #else
333 PNET_BUFFER_LIST nbl;
334 PNET_BUFFER packet;
335 #endif
336 PMDL mdl_head, mdl_tail, curr_mdl;
337 PUCHAR header_va;
338 ULONG out_remaining;
339 ULONG header_extra;
340 shared_buffer_t *header_buf;
341 ULONG outstanding;
342 #if NTDDI_VERSION < NTDDI_VISTA
343 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
344 //UINT packet_length;
345 #else
346 NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csum_info;
347 #endif
348 //FUNCTION_ENTER();
350 #if NTDDI_VERSION < NTDDI_VISTA
351 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
352 if (status != NDIS_STATUS_SUCCESS) {
353 FUNCTION_MSG("No free packets\n");
354 return FALSE;
355 }
357 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
358 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
359 #else
360 nbl = NdisAllocateNetBufferList(xi->rx_nbl_pool, 0, 0);
361 if (!nbl) {
362 /* buffers will be freed in MakePackets */
363 FUNCTION_MSG("No free nbls\n");
364 //FUNCTION_EXIT();
365 return FALSE;
366 }
368 packet = NdisAllocateNetBuffer(xi->rx_packet_pool, NULL, 0, 0);
369 if (!packet) {
370 FUNCTION_MSG("No free packets\n");
371 NdisFreeNetBufferList(nbl);
372 //FUNCTION_EXIT();
373 return FALSE;
374 }
375 #endif
377 if ((!pi->first_mdl->Next || (xi->config_rx_coalesce && pi->total_length <= PAGE_SIZE)) && !pi->split_required) {
378 /* a single buffer <= MTU */
379 header_buf = NULL;
380 /* get all the packet into the header */
381 XenNet_BuildHeader(pi, pi->first_mdl_virtual, PAGE_SIZE);
383 /* have to create a partial mdl over the pb MDL as the pb mdl has a Next which breaks things */
384 curr_mdl = IoAllocateMdl(pi->first_mdl_virtual, pi->total_length, FALSE, FALSE, NULL);
385 XN_ASSERT(curr_mdl);
386 IoBuildPartialMdl(pi->first_mdl, curr_mdl, pi->first_mdl_virtual, pi->total_length);
387 #if NTDDI_VERSION < NTDDI_VISTA
388 NdisChainBufferAtBack(packet, curr_mdl);
389 PACKET_FIRST_PB(packet) = pi->first_pb;
390 #else
391 NET_BUFFER_FIRST_MDL(packet) = curr_mdl;
392 NET_BUFFER_CURRENT_MDL(packet) = curr_mdl;
393 NET_BUFFER_CURRENT_MDL_OFFSET(packet) = 0;
394 NET_BUFFER_DATA_OFFSET(packet) = 0;
395 NET_BUFFER_DATA_LENGTH(packet) = pi->total_length;
396 NB_FIRST_PB(packet) = pi->first_pb;
397 #endif
398 ref_pb(xi, pi->first_pb);
399 } else {
400 XN_ASSERT(ndis_os_minor_version >= 1);
401 header_buf = get_hb_from_freelist(xi);
402 if (!header_buf) {
403 FUNCTION_MSG("No free header buffers\n");
404 #if NTDDI_VERSION < NTDDI_VISTA
405 NdisUnchainBufferAtFront(packet, &curr_mdl);
406 NdisFreePacket(packet);
407 #else
408 NdisFreeNetBufferList(nbl);
409 NdisFreeNetBuffer(packet);
410 #endif
411 return FALSE;
412 }
413 header_va = (PUCHAR)(header_buf + 1);
414 NdisMoveMemory(header_va, pi->header, pi->header_length);
415 //if (pi->ip_proto == 50) {
416 // FUNCTION_MSG("header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead);
417 // FUNCTION_MSG("ip4_header_length = %d\n", pi->ip4_header_length);
418 // FUNCTION_MSG("tcp_header_length = %d\n", pi->tcp_header_length);
419 //}
420 /* make sure only the header is in the first buffer (or the entire packet, but that is done in the above case) */
421 XenNet_BuildHeader(pi, header_va, MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
422 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
423 XN_ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
424 header_buf->mdl->ByteCount = pi->header_length;
425 mdl_head = mdl_tail = curr_mdl = header_buf->mdl;
426 XN_ASSERT(!header_buf->mdl->Next);
427 #if NTDDI_VERSION < NTDDI_VISTA
428 PACKET_FIRST_PB(packet) = header_buf;
429 header_buf->next = pi->curr_pb;
430 NdisChainBufferAtBack(packet, mdl_head);
431 #else
432 NB_FIRST_PB(packet) = header_buf;
433 header_buf->next = pi->curr_pb;
434 NET_BUFFER_FIRST_MDL(packet) = mdl_head;
435 NET_BUFFER_CURRENT_MDL(packet) = mdl_head;
436 NET_BUFFER_CURRENT_MDL_OFFSET(packet) = 0;
437 NET_BUFFER_DATA_OFFSET(packet) = 0;
438 NET_BUFFER_DATA_LENGTH(packet) = pi->header_length;
439 #endif
441 if (pi->split_required) {
442 /* must be ip4 */
443 ULONG tcp_length;
444 USHORT new_ip4_length;
445 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
446 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
447 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
448 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
449 pi->tcp_seq += tcp_length;
450 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
451 /* part of the packet is already present in the header buffer for lookahead */
452 out_remaining = tcp_length - header_extra;
453 XN_ASSERT((LONG)out_remaining >= 0);
454 } else {
455 out_remaining = pi->total_length - pi->header_length;
456 XN_ASSERT((LONG)out_remaining >= 0);
457 }
459 while (out_remaining != 0) {
460 //ULONG in_buffer_offset;
461 ULONG in_buffer_length;
462 ULONG out_length;
464 //if (pi->ip_proto == 50) {
465 // FUNCTION_MSG("in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb);
466 //}
467 if (!pi->curr_mdl || !pi->curr_pb) {
468 FUNCTION_MSG("out of buffers for packet\n");
469 //KdPrint((__DRIVER_NAME " out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb));
470 // TODO: free some stuff or we'll leak
471 /* unchain buffers then free packet */
472 //FUNCTION_EXIT();
473 return FALSE;
474 }
476 in_buffer_length = MmGetMdlByteCount(pi->curr_mdl);
477 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
478 curr_mdl = IoAllocateMdl((PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length, FALSE, FALSE, NULL);
479 XN_ASSERT(curr_mdl);
480 IoBuildPartialMdl(pi->curr_mdl, curr_mdl, (PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length);
481 mdl_tail->Next = curr_mdl;
482 mdl_tail = curr_mdl;
483 curr_mdl->Next = NULL; /* I think this might be redundant */
484 #if NTDDI_VERSION < NTDDI_VISTA
485 #else
486 NET_BUFFER_DATA_LENGTH(packet) += out_length;
487 #endif
488 ref_pb(xi, pi->curr_pb);
489 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
490 if (pi->curr_mdl_offset == in_buffer_length) {
491 pi->curr_mdl = pi->curr_mdl->Next;
492 pi->curr_pb = pi->curr_pb->next;
493 pi->curr_mdl_offset = 0;
494 }
495 out_remaining -= out_length;
496 }
497 #if NTDDI_VERSION < NTDDI_VISTA
498 if (pi->split_required) {
499 // TODO: only if Ip checksum is disabled...
500 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
501 }
502 #endif
503 if (header_extra > 0)
504 pi->header_length -= header_extra;
505 }
507 rc->packet_count++;
508 #if NTDDI_VERSION < NTDDI_VISTA
509 #else
510 NET_BUFFER_LIST_FIRST_NB(nbl) = packet;
511 #endif
513 if (pi->parse_result == PARSE_OK) {
514 #if NTDDI_VERSION < NTDDI_VISTA
515 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
516 packet, TcpIpChecksumPacketInfo);
517 csum_info->Value = 0;
518 if (pi->csum_blank || pi->data_validated || pi->split_required) {
519 BOOLEAN checksum_offload = FALSE;
520 /* we know this is IPv4, and we know Linux always validates the IPv4 checksum for us */
521 if (xi->setting_csum.V4Receive.IpChecksum) {
522 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported) {
523 if (XenNet_CheckIpHeaderSum(pi->header, pi->ip4_header_length))
524 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
525 else
526 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
527 }
528 }
529 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6) {
530 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported) {
531 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
532 checksum_offload = TRUE;
533 }
534 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17) {
535 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
536 checksum_offload = TRUE;
537 }
538 if (pi->csum_blank && (!xi->config_csum_rx_dont_fix || !checksum_offload)) {
539 XenNet_SumPacketData(pi, packet, TRUE);
540 }
541 } else if (xi->config_csum_rx_check && pi->ip_version == 4) {
542 if (xi->setting_csum.V4Receive.IpChecksum) {
543 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported) {
544 if (XenNet_CheckIpHeaderSum(pi->header, pi->ip4_header_length))
545 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
546 else
547 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
548 }
549 }
550 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6) {
551 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported) {
552 if (XenNet_SumPacketData(pi, packet, FALSE)) {
553 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
554 } else {
555 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
556 }
557 }
558 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17) {
559 if (XenNet_SumPacketData(pi, packet, FALSE)) {
560 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
561 } else {
562 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
563 }
564 }
565 }
566 #else
567 csum_info.Value = 0;
568 if (pi->csum_blank || pi->data_validated || pi->mss) {
569 if (pi->ip_proto == 6) {
570 csum_info.Receive.IpChecksumSucceeded = TRUE;
571 csum_info.Receive.TcpChecksumSucceeded = TRUE;
572 } else if (pi->ip_proto == 17) {
573 csum_info.Receive.IpChecksumSucceeded = TRUE;
574 csum_info.Receive.UdpChecksumSucceeded = TRUE;
575 }
576 }
577 NET_BUFFER_LIST_INFO(nbl, TcpIpChecksumNetBufferListInfo) = csum_info.Value;
578 #endif
579 }
581 #if NTDDI_VERSION < NTDDI_VISTA
582 if (!rc->first_packet) {
583 rc->first_packet = packet;
584 } else {
585 PACKET_NEXT_PACKET(rc->last_packet) = packet;
586 }
587 rc->last_packet = packet;
588 rc->packet_count++;
589 #else
590 if (!rc->first_nbl) {
591 rc->first_nbl = nbl;
592 } else {
593 NET_BUFFER_LIST_NEXT_NBL(rc->last_nbl) = nbl;
594 }
595 rc->last_nbl = nbl;
596 NET_BUFFER_LIST_NEXT_NBL(nbl) = NULL;
597 rc->nbl_count++;
598 if (pi->is_multicast) {
599 /* multicast */
600 xi->stats.ifHCInMulticastPkts++;
601 xi->stats.ifHCInMulticastOctets += NET_BUFFER_DATA_LENGTH(packet);
602 } else if (pi->is_broadcast) {
603 /* broadcast */
604 xi->stats.ifHCInBroadcastPkts++;
605 xi->stats.ifHCInBroadcastOctets += NET_BUFFER_DATA_LENGTH(packet);
606 } else {
607 /* unicast */
608 xi->stats.ifHCInUcastPkts++;
609 xi->stats.ifHCInUcastOctets += NET_BUFFER_DATA_LENGTH(packet);
610 }
611 #endif
613 outstanding = InterlockedIncrement(&xi->rx_outstanding);
614 #if NTDDI_VERSION < NTDDI_VISTA
615 if (outstanding > RX_PACKET_HIGH_WATER_MARK || !xi->rx_pb_free) {
616 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
617 } else {
618 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
619 }
620 #if 0
621 /* windows gets lazy about ack packets and holds on to them forever under high load situations. we don't like this */
622 NdisQueryPacketLength(packet, &packet_length);
623 if (pi->parse_result != PARSE_OK || (pi->ip_proto == 6 && packet_length <= NDIS_STATUS_RESOURCES_MAX_LENGTH))
624 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
625 else
626 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
627 #endif
628 #endif
630 //FUNCTION_EXIT();
631 return TRUE;
632 }
634 static VOID
635 XenNet_MakePackets(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi) {
636 UCHAR tcp_flags;
637 shared_buffer_t *page_buf;
639 XenNet_ParsePacketHeader(pi, NULL, XN_HDR_SIZE + xi->current_lookahead);
641 if (!XenNet_FilterAcceptPacket(xi, pi)) {
642 goto done;
643 }
645 if (pi->split_required) {
646 #if NTDDI_VERSION < NTDDI_VISTA
647 /* need to split to mss for NDIS5 */
648 #else
649 switch (xi->current_gso_rx_split_type) {
650 case RX_LSO_SPLIT_HALF:
651 pi->mss = max((pi->tcp_length + 1) / 2, pi->mss);
652 break;
653 case RX_LSO_SPLIT_NONE:
654 pi->mss = 65535;
655 break;
656 }
657 #endif
658 }
660 switch (pi->ip_proto) {
661 case 6: // TCP
662 if (pi->split_required)
663 break;
664 /* fall through */
665 case 17: // UDP
666 if (!XenNet_MakePacket(xi, rc, pi)) {
667 FUNCTION_MSG("Failed to make packet\n");
668 #if NTDDI_VERSION < NTDDI_VISTA
669 xi->stat_rx_no_buffer++;
670 #else
671 xi->stats.ifInDiscards++;
672 #endif
673 goto done;
674 }
675 goto done;
676 default:
677 if (!XenNet_MakePacket(xi, rc, pi)) {
678 FUNCTION_MSG("Failed to make packet\n");
679 #if NTDDI_VERSION < NTDDI_VISTA
680 xi->stat_rx_no_buffer++;
681 #else
682 xi->stats.ifInDiscards++;
683 #endif
684 goto done;
685 }
686 goto done;
687 }
689 /* this is the split_required code */
690 pi->tcp_remaining = pi->tcp_length;
692 /* we can make certain assumptions here as the following code is only for tcp4 */
693 tcp_flags = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13];
694 /* clear all tcp flags except ack except for the last packet */
695 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] &= 0x10;
696 while (pi->tcp_remaining) {
697 if (pi->tcp_remaining <= pi->mss) {
698 /* restore tcp flags for the last packet */
699 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] = tcp_flags;
700 }
701 if (!XenNet_MakePacket(xi, rc, pi)) {
702 FUNCTION_MSG("Failed to make packet\n");
703 #if NTDDI_VERSION < NTDDI_VISTA
704 xi->stat_rx_no_buffer++;
705 #else
706 xi->stats.ifInDiscards++;
707 #endif
708 break; /* we are out of memory - just drop the packets */
709 }
710 }
711 done:
712 page_buf = pi->first_pb;
713 while (page_buf) {
714 shared_buffer_t *next_pb = page_buf->next;
715 put_pb_on_freelist(xi, page_buf); /* this doesn't actually free the page_puf if there are outstanding references */
716 page_buf = next_pb;
717 }
718 XenNet_ClearPacketInfo(pi);
719 //FUNCTION_EXIT();
720 return;
721 }
723 #if NTDDI_VERSION < NTDDI_VISTA
724 /* called at DISPATCH_LEVEL */
725 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
726 VOID
727 XenNet_ReturnPacket(NDIS_HANDLE adapter_context, PNDIS_PACKET packet) {
728 struct xennet_info *xi = adapter_context;
729 PNDIS_BUFFER buffer;
730 shared_buffer_t *page_buf = PACKET_FIRST_PB(packet);
732 //FUNCTION_ENTER();
733 NdisUnchainBufferAtFront(packet, &buffer);
735 while (buffer) {
736 shared_buffer_t *next_buf;
737 XN_ASSERT(page_buf);
738 next_buf = page_buf->next;
739 if (!page_buf->virtual) {
740 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
741 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(buffer) - 1);
742 } else {
743 if (buffer != page_buf->mdl)
744 NdisFreeBuffer(buffer);
745 put_pb_on_freelist(xi, page_buf);
746 }
747 NdisUnchainBufferAtFront(packet, &buffer);
748 page_buf = next_buf;
749 }
751 NdisFreePacket(packet);
752 InterlockedDecrement(&xi->rx_outstanding);
753 if (!xi->rx_outstanding && xi->device_state != DEVICE_STATE_ACTIVE)
754 KeSetEvent(&xi->rx_idle_event, IO_NO_INCREMENT, FALSE);
755 //FUNCTION_EXIT();
756 }
757 #else
758 /* called at <= DISPATCH_LEVEL */
759 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
760 VOID
761 XenNet_ReturnNetBufferLists(NDIS_HANDLE adapter_context, PNET_BUFFER_LIST curr_nbl, ULONG return_flags) {
762 struct xennet_info *xi = adapter_context;
763 UNREFERENCED_PARAMETER(return_flags);
765 //FUNCTION_ENTER();
767 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
769 XN_ASSERT(xi);
770 while (curr_nbl) {
771 PNET_BUFFER_LIST next_nbl;
772 PNET_BUFFER curr_nb;
774 next_nbl = NET_BUFFER_LIST_NEXT_NBL(curr_nbl);
775 curr_nb = NET_BUFFER_LIST_FIRST_NB(curr_nbl);
776 while (curr_nb) {
777 PNET_BUFFER next_nb;
778 PMDL curr_mdl;
779 shared_buffer_t *page_buf;
781 next_nb = NET_BUFFER_NEXT_NB(curr_nb);
782 curr_mdl = NET_BUFFER_FIRST_MDL(curr_nb);
783 page_buf = NB_FIRST_PB(curr_nb);
784 while (curr_mdl) {
785 shared_buffer_t *next_buf;
786 PMDL next_mdl;
788 XN_ASSERT(page_buf); /* make sure that there is a pb to match this mdl */
789 next_mdl = curr_mdl->Next;
790 next_buf = page_buf->next;
791 if (!page_buf->virtual) {
792 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
793 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(curr_mdl) - 1);
794 } else {
795 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
796 if (curr_mdl != page_buf->mdl) {
797 //KdPrint((__DRIVER_NAME " curr_mdl = %p, page_buf->mdl = %p\n", curr_mdl, page_buf->mdl));
798 IoFreeMdl(curr_mdl);
799 }
800 put_pb_on_freelist(xi, page_buf);
801 }
802 curr_mdl = next_mdl;
803 page_buf = next_buf;
804 }
806 NdisFreeNetBuffer(curr_nb);
807 InterlockedDecrement(&xi->rx_outstanding);
809 curr_nb = next_nb;
810 }
811 NdisFreeNetBufferList(curr_nbl);
812 curr_nbl = next_nbl;
813 }
815 if (!xi->rx_outstanding && xi->device_state != DEVICE_STATE_ACTIVE)
816 KeSetEvent(&xi->rx_idle_event, IO_NO_INCREMENT, FALSE);
818 //FUNCTION_EXIT();
819 }
820 #endif
822 /* We limit the number of packets per interrupt so that acks get a chance
823 under high rx load. The DPC is immediately re-scheduled */
825 #define MAXIMUM_PACKETS_PER_INDICATE 32
827 #define MAXIMUM_PACKETS_PER_INTERRUPT 2560 /* this is calculated before large packet split */
828 #define MAXIMUM_DATA_PER_INTERRUPT (MAXIMUM_PACKETS_PER_INTERRUPT * 1500) /* help account for large packets */
830 // Called at DISPATCH_LEVEL
831 BOOLEAN
832 XenNet_RxBufferCheck(struct xennet_info *xi) {
833 RING_IDX cons, prod;
834 ULONG packet_count = 0;
835 ULONG packet_data = 0;
836 ULONG buffer_count = 0;
837 USHORT id;
838 int more_to_do = FALSE;
839 shared_buffer_t *page_buf;
840 #if NTDDI_VERSION < NTDDI_VISTA
841 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
842 PNDIS_PACKET first_header_only_packet;
843 PNDIS_PACKET last_header_only_packet;
844 #else
845 #endif
846 //ULONG nbl_count = 0;
847 ULONG interim_packet_data = 0;
848 struct netif_extra_info *ei;
849 rx_context_t rc;
850 packet_info_t *pi = &xi->rxpi[KeGetCurrentProcessorNumber() & 0xff];
851 shared_buffer_t *head_buf = NULL;
852 shared_buffer_t *tail_buf = NULL;
853 shared_buffer_t *last_buf = NULL;
854 BOOLEAN extra_info_flag = FALSE;
855 BOOLEAN more_data_flag = FALSE;
856 BOOLEAN dont_set_event;
857 //FUNCTION_ENTER();
859 #if NTDDI_VERSION < NTDDI_VISTA
860 rc.first_packet = NULL;
861 rc.last_packet = NULL;
862 rc.packet_count = 0;
863 #else
864 rc.first_nbl = NULL;
865 rc.last_nbl = NULL;
866 rc.packet_count = 0;
867 rc.nbl_count = 0;
868 #endif
870 /* get all the buffers off the ring as quickly as possible so the lock is held for a minimum amount of time */
871 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
873 if (xi->device_state != DEVICE_STATE_ACTIVE) {
874 /* there is a chance that our Dpc had been queued just before the shutdown... */
875 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
876 return FALSE;
877 }
879 if (xi->rx_partial_buf) {
880 head_buf = xi->rx_partial_buf;
881 tail_buf = xi->rx_partial_buf;
882 while (tail_buf->next)
883 tail_buf = tail_buf->next;
884 more_data_flag = xi->rx_partial_more_data_flag;
885 extra_info_flag = xi->rx_partial_extra_info_flag;
886 xi->rx_partial_buf = NULL;
887 }
889 do {
890 prod = xi->rx_ring.sring->rsp_prod;
891 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
893 for (cons = xi->rx_ring.rsp_cons; cons != prod && packet_count < MAXIMUM_PACKETS_PER_INTERRUPT && packet_data < MAXIMUM_DATA_PER_INTERRUPT; cons++) {
894 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
895 page_buf = xi->rx_ring_pbs[id];
896 XN_ASSERT(page_buf);
897 xi->rx_ring_pbs[id] = NULL;
898 xi->rx_id_free++;
899 memcpy(&page_buf->rsp, RING_GET_RESPONSE(&xi->rx_ring, cons), max(sizeof(struct netif_rx_response), sizeof(struct netif_extra_info)));
900 if (!extra_info_flag) {
901 if (page_buf->rsp.status <= 0 || page_buf->rsp.offset + page_buf->rsp.status > PAGE_SIZE) {
902 FUNCTION_MSG("Error: rsp offset %d, size %d\n",
903 page_buf->rsp.offset, page_buf->rsp.status);
904 XN_ASSERT(!extra_info_flag);
905 put_pb_on_freelist(xi, page_buf);
906 continue;
907 }
908 }
910 if (!head_buf) {
911 head_buf = page_buf;
912 tail_buf = page_buf;
913 } else {
914 tail_buf->next = page_buf;
915 tail_buf = page_buf;
916 }
917 page_buf->next = NULL;
919 if (extra_info_flag) {
920 ei = (struct netif_extra_info *)&page_buf->rsp;
921 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
922 } else {
923 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
924 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
925 interim_packet_data += page_buf->rsp.status;
926 }
928 if (!extra_info_flag && !more_data_flag) {
929 last_buf = page_buf;
930 packet_count++;
931 packet_data += interim_packet_data;
932 interim_packet_data = 0;
933 }
934 buffer_count++;
935 }
936 xi->rx_ring.rsp_cons = cons;
938 /* Give netback more buffers */
939 XenNet_FillRing(xi);
941 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
942 break;
944 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx_ring);
945 if (!more_to_do) {
946 xi->rx_ring.sring->rsp_event = xi->rx_ring.rsp_cons + 1;
947 KeMemoryBarrier();
948 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx_ring);
949 }
950 } while (more_to_do);
952 /* anything past last_buf belongs to an incomplete packet... */
953 if (last_buf && last_buf->next)
954 {
955 FUNCTION_MSG("Partial receive\n");
956 xi->rx_partial_buf = last_buf->next;
957 xi->rx_partial_more_data_flag = more_data_flag;
958 xi->rx_partial_extra_info_flag = extra_info_flag;
959 last_buf->next = NULL;
960 }
962 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
964 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
965 {
966 /* fire again immediately */
967 FUNCTION_MSG("Dpc Duration Exceeded\n");
968 /* we want the Dpc on the end of the queue. By definition we are already on the right CPU so we know the Dpc queue will be run immediately */
969 // KeSetImportanceDpc(&xi->rxtx_dpc, MediumImportance);
970 KeInsertQueueDpc(&xi->rxtx_dpc, NULL, NULL);
971 /* dont set an event in TX path */
972 dont_set_event = TRUE;
973 }
974 else
975 {
976 /* make sure the Dpc queue is run immediately next interrupt */
977 // KeSetImportanceDpc(&xi->rxtx_dpc, HighImportance);
978 /* set an event in TX path */
979 dont_set_event = FALSE;
980 }
982 /* make packets out of the buffers */
983 page_buf = head_buf;
984 extra_info_flag = FALSE;
985 more_data_flag = FALSE;
987 while (page_buf) {
988 shared_buffer_t *next_buf = page_buf->next;
989 PMDL mdl;
991 page_buf->next = NULL;
992 if (extra_info_flag) {
993 //KdPrint((__DRIVER_NAME " processing extra info\n"));
994 ei = (struct netif_extra_info *)&page_buf->rsp;
995 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
996 switch (ei->type)
997 {
998 case XEN_NETIF_EXTRA_TYPE_GSO:
999 switch (ei->u.gso.type) {
1000 case XEN_NETIF_GSO_TYPE_TCPV4:
1001 pi->mss = ei->u.gso.size;
1002 // TODO - put this assertion somewhere XN_ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
1003 break;
1004 default:
1005 FUNCTION_MSG("Unknown GSO type (%d) detected\n", ei->u.gso.type);
1006 break;
1008 break;
1009 default:
1010 FUNCTION_MSG("Unknown extra info type (%d) detected\n", ei->type);
1011 break;
1013 put_pb_on_freelist(xi, page_buf);
1014 } else {
1015 XN_ASSERT(!page_buf->rsp.offset);
1016 if (!more_data_flag) { // handling the packet's 1st buffer
1017 if (page_buf->rsp.flags & NETRXF_csum_blank)
1018 pi->csum_blank = TRUE;
1019 if (page_buf->rsp.flags & NETRXF_data_validated)
1020 pi->data_validated = TRUE;
1022 mdl = page_buf->mdl;
1023 mdl->ByteCount = page_buf->rsp.status; //NdisAdjustBufferLength(mdl, page_buf->rsp.status);
1024 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
1025 if (pi->first_pb) {
1026 XN_ASSERT(pi->curr_pb);
1027 //KdPrint((__DRIVER_NAME " additional buffer\n"));
1028 pi->curr_pb->next = page_buf;
1029 pi->curr_pb = page_buf;
1030 XN_ASSERT(pi->curr_mdl);
1031 pi->curr_mdl->Next = mdl;
1032 pi->curr_mdl = mdl;
1033 } else {
1034 pi->first_pb = page_buf;
1035 pi->curr_pb = page_buf;
1036 pi->first_mdl = mdl;
1037 pi->curr_mdl = mdl;
1039 //pi->mdl_count++;
1040 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
1041 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
1042 pi->total_length = pi->total_length + page_buf->rsp.status;
1045 /* Packet done, add it to the list */
1046 if (!more_data_flag && !extra_info_flag) {
1047 pi->curr_pb = pi->first_pb;
1048 pi->curr_mdl = pi->first_mdl;
1049 XenNet_MakePackets(xi, &rc, pi);
1052 page_buf = next_buf;
1054 XN_ASSERT(!more_data_flag && !extra_info_flag);
1056 #if NTDDI_VERSION < NTDDI_VISTA
1057 packet_count = 0;
1058 first_header_only_packet = NULL;
1059 last_header_only_packet = NULL;
1061 while (rc.first_packet) {
1062 PNDIS_PACKET packet;
1063 NDIS_STATUS status;
1065 packet = rc.first_packet;
1066 XN_ASSERT(PACKET_FIRST_PB(packet));
1067 rc.first_packet = PACKET_NEXT_PACKET(packet);
1068 status = NDIS_GET_PACKET_STATUS(packet);
1069 if (status == NDIS_STATUS_RESOURCES) {
1070 if (!first_header_only_packet) {
1071 first_header_only_packet = packet;
1072 } else {
1073 PACKET_NEXT_PACKET(last_header_only_packet) = packet;
1075 last_header_only_packet = packet;
1076 PACKET_NEXT_PACKET(packet) = NULL;
1078 packets[packet_count++] = packet;
1079 /* if we indicate a packet with NDIS_STATUS_RESOURCES then any following packet can't be NDIS_STATUS_SUCCESS */
1080 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || !rc.first_packet
1081 || (NDIS_GET_PACKET_STATUS(rc.first_packet) == NDIS_STATUS_SUCCESS
1082 && status == NDIS_STATUS_RESOURCES)) {
1083 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
1084 packet_count = 0;
1087 /* now return the packets for which we indicated NDIS_STATUS_RESOURCES */
1088 while (first_header_only_packet) {
1089 PNDIS_PACKET packet = first_header_only_packet;
1090 first_header_only_packet = PACKET_NEXT_PACKET(packet);
1091 XenNet_ReturnPacket(xi, packet);
1093 #else
1094 if (rc.first_nbl) {
1095 NdisMIndicateReceiveNetBufferLists(xi->adapter_handle, rc.first_nbl,
1096 NDIS_DEFAULT_PORT_NUMBER, rc.nbl_count,
1097 NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL
1098 //| NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE
1099 | NDIS_RECEIVE_FLAGS_PERFECT_FILTERED);
1101 #endif
1102 //FUNCTION_EXIT();
1103 return dont_set_event;
1106 static VOID
1107 XenNet_BufferFree(xennet_info_t *xi)
1109 shared_buffer_t *sb;
1110 int i;
1112 for (i = 0; i < NET_RX_RING_SIZE; i++) {
1113 if (xi->rx_ring_pbs[i] != NULL) {
1114 put_pb_on_freelist(xi, xi->rx_ring_pbs[i]);
1115 xi->rx_ring_pbs[i] = NULL;
1119 /* because we are shutting down this won't allocate new ones */
1120 while ((sb = get_pb_from_freelist(xi)) != NULL) {
1121 XnEndAccess(xi->handle,
1122 sb->gref, FALSE, (ULONG)'XNRX');
1123 IoFreeMdl(sb->mdl);
1124 ExFreePoolWithTag(sb->virtual, XENNET_POOL_TAG);
1125 ExFreePoolWithTag(sb, XENNET_POOL_TAG);
1127 while ((sb = get_hb_from_freelist(xi)) != NULL) {
1128 IoFreeMdl(sb->mdl);
1129 ExFreePoolWithTag(sb, XENNET_POOL_TAG);
1133 BOOLEAN
1134 XenNet_RxInit(xennet_info_t *xi) {
1135 #if NTDDI_VERSION < NTDDI_VISTA
1136 NDIS_STATUS status;
1137 #else
1138 NET_BUFFER_LIST_POOL_PARAMETERS nbl_pool_parameters;
1139 NET_BUFFER_POOL_PARAMETERS nb_pool_parameters;
1140 #endif
1141 int ret;
1142 int i;
1144 FUNCTION_ENTER();
1146 // this stuff needs to be done once only...
1147 KeInitializeSpinLock(&xi->rx_lock);
1148 KeInitializeEvent(&xi->rx_idle_event, SynchronizationEvent, FALSE);
1149 #if NTDDI_VERSION < NTDDI_VISTA
1150 FUNCTION_MSG("NdisSystemProcessorCount = %d\n", NdisSystemProcessorCount());
1151 xi->rxpi = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(packet_info_t) * NdisSystemProcessorCount(), XENNET_POOL_TAG, NormalPoolPriority);
1152 #else
1153 FUNCTION_MSG("KeQueryActiveProcessorCount = %d\n", KeQueryActiveProcessorCount(NULL));
1154 xi->rxpi = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(packet_info_t) * KeQueryActiveProcessorCount(NULL), XENNET_POOL_TAG, NormalPoolPriority);
1155 #endif
1156 if (!xi->rxpi) {
1157 FUNCTION_MSG("ExAllocatePoolWithTagPriority failed\n");
1158 return FALSE;
1160 #if NTDDI_VERSION < NTDDI_VISTA
1161 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount());
1162 #else
1163 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * KeQueryActiveProcessorCount(NULL));
1164 #endif
1165 ret = stack_new(&xi->rx_pb_stack, NET_RX_RING_SIZE * 4);
1166 if (!ret) {
1167 FUNCTION_MSG("Failed to allocate rx_pb_stack\n");
1168 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1169 return FALSE;
1171 ret = stack_new(&xi->rx_hb_stack, NET_RX_RING_SIZE * 4);
1172 if (!ret) {
1173 FUNCTION_MSG("Failed to allocate rx_hb_stack\n");
1174 stack_delete(xi->rx_pb_stack, NULL, NULL);
1175 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1176 return FALSE;
1179 xi->rx_id_free = NET_RX_RING_SIZE;
1180 xi->rx_outstanding = 0;
1182 for (i = 0; i < NET_RX_RING_SIZE; i++) {
1183 xi->rx_ring_pbs[i] = NULL;
1186 #if NTDDI_VERSION < NTDDI_VISTA
1187 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4, PROTOCOL_RESERVED_SIZE_IN_PACKET);
1188 if (status != NDIS_STATUS_SUCCESS) {
1189 FUNCTION_MSG("NdisAllocatePacketPool failed with 0x%x\n", status);
1190 return FALSE;
1192 #else
1193 nbl_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
1194 nbl_pool_parameters.Header.Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
1195 nbl_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
1196 nbl_pool_parameters.ProtocolId = NDIS_PROTOCOL_ID_DEFAULT;
1197 nbl_pool_parameters.fAllocateNetBuffer = FALSE;
1198 nbl_pool_parameters.ContextSize = 0;
1199 nbl_pool_parameters.PoolTag = XENNET_POOL_TAG;
1200 nbl_pool_parameters.DataSize = 0; /* NET_BUFFERS are always allocated separately */
1202 xi->rx_nbl_pool = NdisAllocateNetBufferListPool(xi->adapter_handle, &nbl_pool_parameters);
1203 if (!xi->rx_nbl_pool) {
1204 FUNCTION_MSG("NdisAllocateNetBufferListPool failed\n");
1205 return FALSE;
1208 nb_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
1209 nb_pool_parameters.Header.Revision = NET_BUFFER_POOL_PARAMETERS_REVISION_1;
1210 nb_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_POOL_PARAMETERS_REVISION_1;
1211 nb_pool_parameters.PoolTag = XENNET_POOL_TAG;
1212 nb_pool_parameters.DataSize = 0; /* the buffers come from the ring */
1213 xi->rx_packet_pool = NdisAllocateNetBufferPool(xi->adapter_handle, &nb_pool_parameters);
1214 if (!xi->rx_packet_pool) {
1215 FUNCTION_MSG("NdisAllocateNetBufferPool (rx_packet_pool) failed\n");
1216 return FALSE;
1218 #endif
1219 XenNet_FillRing(xi);
1221 FUNCTION_EXIT();
1223 return TRUE;
1226 VOID
1227 XenNet_RxShutdown(xennet_info_t *xi) {
1228 KIRQL old_irql;
1229 UNREFERENCED_PARAMETER(xi);
1231 FUNCTION_ENTER();
1233 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1234 while (xi->rx_outstanding) {
1235 FUNCTION_MSG("Waiting for %d packets to be returned\n", xi->rx_outstanding);
1236 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1237 KeWaitForSingleObject(&xi->rx_idle_event, Executive, KernelMode, FALSE, NULL);
1238 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1240 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1242 XenNet_BufferFree(xi);
1244 stack_delete(xi->rx_pb_stack, NULL, NULL);
1245 stack_delete(xi->rx_hb_stack, NULL, NULL);
1248 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1250 #if NTDDI_VERSION < NTDDI_VISTA
1251 NdisFreePacketPool(xi->rx_packet_pool);
1252 #else
1253 NdisFreeNetBufferPool(xi->rx_packet_pool);
1254 NdisFreeNetBufferListPool(xi->rx_nbl_pool);
1255 #endif
1257 FUNCTION_EXIT();
1258 return;