win-pvdrivers

view xennet/xennet_rx.c @ 1067:2ef536c2d9fe

Fix packet exhaustion issue in 2003 where windows doesn't return packets
author James Harper <james.harper@bendigoit.com.au>
date Tue Oct 29 19:39:52 2013 +1100 (2013-10-29)
parents fba0ce4d9e54
children ebfa9417f1ee
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 #define RX_STATS
25 #ifdef RX_STATS
26 static ULONG x_packet_allocated = 0;
27 static ULONG x_packet_freed = 0;
28 static ULONG x_packet_resources = 0;
30 static ULONG x_0_64_packet_indicated = 0;
31 static ULONG x_65_128_packet_indicated = 0;
32 static ULONG x_129_256_packet_indicated = 0;
33 static ULONG x_257_512_packet_indicated = 0;
34 static ULONG x_513_1024_packet_indicated = 0;
35 static ULONG x_1025_1514_packet_indicated = 0;
36 static ULONG x_1515_65535_packet_indicated = 0;
37 static ULONG x_0_64_packet_returned = 0;
38 static ULONG x_65_128_packet_returned = 0;
39 static ULONG x_129_256_packet_returned = 0;
40 static ULONG x_257_512_packet_returned = 0;
41 static ULONG x_513_1024_packet_returned = 0;
42 static ULONG x_1025_1514_packet_returned = 0;
43 static ULONG x_1515_65535_packet_returned = 0;
45 static ULONG x_1_frag_packet_indicated = 0;
46 static ULONG x_2_frag_packet_indicated = 0;
47 static ULONG x_3_frag_packet_indicated = 0;
48 static ULONG x_4_or_more_frag_packet_indicated = 0;
49 static ULONG x_1_frag_packet_returned = 0;
50 static ULONG x_2_frag_packet_returned = 0;
51 static ULONG x_3_frag_packet_returned = 0;
52 static ULONG x_4_or_more_frag_packet_returned = 0;
54 static LARGE_INTEGER x_next_print_time = {0};
56 static VOID
57 dump_x_stats() {
58 FUNCTION_MSG("x_pkt_allocated (outstanding, resources) = %d (%d, %d)\n", x_packet_allocated, x_packet_allocated - x_packet_freed, x_packet_resources);
59 FUNCTION_MSG("x_0_64_pkt_indicated (outstanding) = %d (%d)\n", x_0_64_packet_indicated, x_0_64_packet_indicated - x_0_64_packet_returned);
60 FUNCTION_MSG("x_65_128_pkt_indicated (outstanding) = %d (%d)\n", x_65_128_packet_indicated, x_65_128_packet_indicated - x_65_128_packet_returned);
61 FUNCTION_MSG("x_129_256_pkt_indicated (outstanding) = %d (%d)\n", x_129_256_packet_indicated, x_129_256_packet_indicated - x_129_256_packet_returned);
62 FUNCTION_MSG("x_257_512_pkt_indicated (outstanding) = %d (%d)\n", x_257_512_packet_indicated, x_257_512_packet_indicated - x_257_512_packet_returned);
63 FUNCTION_MSG("x_513_1024_pkt_indicated (outstanding) = %d (%d)\n", x_513_1024_packet_indicated, x_513_1024_packet_indicated - x_513_1024_packet_returned);
64 FUNCTION_MSG("x_1025_1514_pkt_indicated (outstanding) = %d (%d)\n", x_1025_1514_packet_indicated, x_1025_1514_packet_indicated - x_1025_1514_packet_returned);
65 FUNCTION_MSG("x_1515_65535_pkt_indicated (outstanding) = %d (%d)\n", x_1515_65535_packet_indicated, x_1515_65535_packet_indicated - x_1515_65535_packet_returned);
66 FUNCTION_MSG("x_1_frag_pkt_indicated (outstanding) = %d (%d)\n", x_1_frag_packet_indicated, x_1_frag_packet_indicated - x_1_frag_packet_returned);
67 FUNCTION_MSG("x_2_frag_pkt_indicated (outstanding) = %d (%d)\n", x_2_frag_packet_indicated, x_2_frag_packet_indicated - x_2_frag_packet_returned);
68 FUNCTION_MSG("x_3_frag_pkt_indicated (outstanding) = %d (%d)\n", x_3_frag_packet_indicated, x_3_frag_packet_indicated - x_3_frag_packet_returned);
69 FUNCTION_MSG("x_4+_frag_pkt_indicated (outstanding) = %d (%d)\n", x_4_or_more_frag_packet_indicated, x_4_or_more_frag_packet_indicated - x_4_or_more_frag_packet_returned);
70 }
71 #endif
73 static __inline shared_buffer_t *
74 get_pb_from_freelist(struct xennet_info *xi)
75 {
76 shared_buffer_t *pb;
77 PVOID ptr_ref;
79 if (stack_pop(xi->rx_pb_stack, &ptr_ref))
80 {
81 pb = ptr_ref;
82 pb->ref_count = 1;
83 InterlockedDecrement(&xi->rx_pb_free);
84 return pb;
85 }
87 /* don't allocate a new one if we are shutting down */
88 if (xi->device_state != DEVICE_STATE_INITIALISING && xi->device_state != DEVICE_STATE_ACTIVE)
89 return NULL;
91 pb = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(shared_buffer_t), XENNET_POOL_TAG, LowPoolPriority);
92 if (!pb)
93 return NULL;
94 pb->virtual = ExAllocatePoolWithTagPriority(NonPagedPool, PAGE_SIZE, XENNET_POOL_TAG, LowPoolPriority);
95 if (!pb->virtual)
96 {
97 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
98 return NULL;
99 }
100 pb->mdl = IoAllocateMdl(pb->virtual, PAGE_SIZE, FALSE, FALSE, NULL);
101 if (!pb->mdl)
102 {
103 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
104 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
105 return NULL;
106 }
107 pb->gref = (grant_ref_t)XnGrantAccess(xi->handle,
108 (ULONG)(MmGetPhysicalAddress(pb->virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
109 if (pb->gref == INVALID_GRANT_REF)
110 {
111 IoFreeMdl(pb->mdl);
112 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
113 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
114 return NULL;
115 }
116 MmBuildMdlForNonPagedPool(pb->mdl);
117 pb->ref_count = 1;
118 return pb;
119 }
121 static __inline VOID
122 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
123 {
124 UNREFERENCED_PARAMETER(xi);
125 InterlockedIncrement(&pb->ref_count);
126 }
128 static __inline VOID
129 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
130 {
131 if (InterlockedDecrement(&pb->ref_count) == 0)
132 {
133 //NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
134 //NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
135 if (xi->rx_pb_free > RX_MAX_PB_FREELIST)
136 {
137 XnEndAccess(xi->handle, pb->gref, FALSE, (ULONG)'XNRX');
138 IoFreeMdl(pb->mdl);
139 ExFreePoolWithTag(pb->virtual, XENNET_POOL_TAG);
140 ExFreePoolWithTag(pb, XENNET_POOL_TAG);
141 return;
142 }
143 pb->mdl->ByteCount = PAGE_SIZE;
144 pb->mdl->Next = NULL;
145 pb->next = NULL;
146 stack_push(xi->rx_pb_stack, pb);
147 InterlockedIncrement(&xi->rx_pb_free);
148 }
149 }
151 static __inline shared_buffer_t *
152 get_hb_from_freelist(struct xennet_info *xi)
153 {
154 shared_buffer_t *hb;
155 PVOID ptr_ref;
157 if (stack_pop(xi->rx_hb_stack, &ptr_ref))
158 {
159 hb = ptr_ref;
160 InterlockedDecrement(&xi->rx_hb_free);
161 return hb;
162 }
164 /* don't allocate a new one if we are shutting down */
165 if (xi->device_state != DEVICE_STATE_INITIALISING && xi->device_state != DEVICE_STATE_ACTIVE)
166 return NULL;
168 hb = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, XENNET_POOL_TAG, LowPoolPriority);
169 if (!hb)
170 return NULL;
171 NdisZeroMemory(hb, sizeof(shared_buffer_t));
172 hb->mdl = IoAllocateMdl(hb + 1, MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, FALSE, FALSE, NULL);
173 if (!hb->mdl) {
174 ExFreePoolWithTag(hb, XENNET_POOL_TAG);
175 return NULL;
176 }
177 MmBuildMdlForNonPagedPool(hb->mdl);
178 return hb;
179 }
181 static __inline VOID
182 put_hb_on_freelist(struct xennet_info *xi, shared_buffer_t *hb)
183 {
184 XN_ASSERT(xi);
185 hb->mdl->ByteCount = sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH;
186 hb->mdl->Next = NULL;
187 hb->next = NULL;
188 stack_push(xi->rx_hb_stack, hb);
189 InterlockedIncrement(&xi->rx_hb_free);
190 }
192 // Called at DISPATCH_LEVEL with rx lock held
193 static VOID
194 XenNet_FillRing(struct xennet_info *xi)
195 {
196 unsigned short id;
197 shared_buffer_t *page_buf;
198 ULONG i, notify;
199 ULONG batch_target;
200 RING_IDX req_prod = xi->rx_ring.req_prod_pvt;
201 netif_rx_request_t *req;
203 //FUNCTION_ENTER();
205 if (xi->device_state != DEVICE_STATE_ACTIVE)
206 return;
208 batch_target = xi->rx_target - (req_prod - xi->rx_ring.rsp_cons);
210 if (batch_target < (xi->rx_target >> 2)) {
211 //FUNCTION_EXIT();
212 return; /* only refill if we are less than 3/4 full already */
213 }
215 for (i = 0; i < batch_target; i++) {
216 page_buf = get_pb_from_freelist(xi);
217 if (!page_buf) {
218 FUNCTION_MSG("Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target);
219 break;
220 }
221 xi->rx_id_free--;
223 /* Give to netback */
224 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
225 XN_ASSERT(xi->rx_ring_pbs[id] == NULL);
226 xi->rx_ring_pbs[id] = page_buf;
227 req = RING_GET_REQUEST(&xi->rx_ring, req_prod + i);
228 req->id = id;
229 req->gref = page_buf->gref;
230 XN_ASSERT(req->gref != INVALID_GRANT_REF);
231 }
232 KeMemoryBarrier();
233 xi->rx_ring.req_prod_pvt = req_prod + i;
234 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx_ring, notify);
235 if (notify) {
236 XnNotify(xi->handle, xi->event_channel);
237 }
239 //FUNCTION_EXIT();
241 return;
242 }
244 #if NTDDI_VERSION < NTDDI_VISTA
245 typedef struct {
246 PNDIS_PACKET first_packet;
247 PNDIS_PACKET last_packet;
248 ULONG packet_count;
249 } rx_context_t;
250 #else
251 typedef struct {
252 PNET_BUFFER_LIST first_nbl;
253 PNET_BUFFER_LIST last_nbl;
254 ULONG packet_count;
255 ULONG nbl_count;
256 } rx_context_t;
257 #endif
259 #if NTDDI_VERSION < NTDDI_VISTA
260 /*
261 NDIS5 appears to insist that the checksum on received packets is correct, and won't
262 believe us when we lie about it, which happens when the packet is generated on the
263 same bridge in Dom0. Doh!
264 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
265 */
267 static BOOLEAN
268 XenNet_SumPacketData(
269 packet_info_t *pi,
270 PNDIS_PACKET packet,
271 BOOLEAN set_csum) {
272 USHORT i;
273 PUCHAR buffer;
274 PMDL mdl;
275 UINT total_length;
276 UINT data_length;
277 UINT buffer_length;
278 USHORT buffer_offset;
279 ULONG csum;
280 PUSHORT csum_ptr;
281 USHORT remaining;
282 USHORT ip4_length;
283 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
285 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
286 if (!buffer) {
287 FUNCTION_MSG("NdisGetFirstBufferFromPacketSafe failed, buffer == NULL\n");
288 return FALSE;
289 }
290 XN_ASSERT(mdl);
292 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
293 data_length = ip4_length + XN_HDR_SIZE;
295 if ((USHORT)data_length > total_length) {
296 FUNCTION_MSG("Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length);
297 return FALSE;
298 }
300 switch (pi->ip_proto) {
301 case 6:
302 XN_ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
303 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
304 break;
305 case 17:
306 XN_ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
307 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
308 break;
309 default:
310 FUNCTION_MSG("Don't know how to calc sum for IP Proto %d\n", pi->ip_proto);
311 //FUNCTION_EXIT();
312 return FALSE; // should never happen
313 }
315 if (set_csum)
316 *csum_ptr = 0;
318 csum = 0;
319 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
320 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
321 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
323 remaining = ip4_length - pi->ip4_header_length;
325 csum += remaining;
327 csum_span = FALSE;
328 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
329 while (i < data_length) {
330 /* don't include the checksum field itself in the calculation */
331 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6)) {
332 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
333 i += 2;
334 buffer_offset += 2;
335 continue;
336 }
337 if (csum_span) {
338 /* the other half of the next bit */
339 XN_ASSERT(buffer_offset == 0);
340 csum += (USHORT)buffer[buffer_offset];
341 csum_span = FALSE;
342 i += 1;
343 buffer_offset += 1;
344 } else if (buffer_offset == buffer_length - 1) {
345 /* deal with a buffer ending on an odd byte boundary */
346 csum += (USHORT)buffer[buffer_offset] << 8;
347 csum_span = TRUE;
348 i += 1;
349 buffer_offset += 1;
350 } else {
351 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
352 i += 2;
353 buffer_offset += 2;
354 }
355 if (buffer_offset == buffer_length && i < total_length) {
356 NdisGetNextBuffer(mdl, &mdl);
357 if (mdl == NULL) {
358 FUNCTION_MSG(__DRIVER_NAME " Ran out of buffers\n");
359 return FALSE; // should never happen
360 }
361 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
362 XN_ASSERT(buffer_length);
363 buffer_offset = 0;
364 }
365 }
367 while (csum & 0xFFFF0000)
368 csum = (csum & 0xFFFF) + (csum >> 16);
370 if (set_csum) {
371 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
372 } else {
373 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
374 }
375 return TRUE;
376 }
377 #endif
379 static BOOLEAN
380 XenNet_MakePacket(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi) {
381 #if NTDDI_VERSION < NTDDI_VISTA
382 NDIS_STATUS status;
383 PNDIS_PACKET packet;
384 #else
385 PNET_BUFFER_LIST nbl;
386 PNET_BUFFER packet;
387 #endif
388 PMDL mdl_head, mdl_tail, curr_mdl;
389 PUCHAR header_va;
390 ULONG out_remaining;
391 ULONG header_extra;
392 shared_buffer_t *header_buf;
393 ULONG outstanding;
394 #if NTDDI_VERSION < NTDDI_VISTA
395 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
396 //UINT packet_length;
397 #else
398 NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csum_info;
399 #endif
400 //FUNCTION_ENTER();
402 #if NTDDI_VERSION < NTDDI_VISTA
403 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
404 if (status != NDIS_STATUS_SUCCESS) {
405 FUNCTION_MSG("No free packets\n");
406 return FALSE;
407 }
409 InterlockedIncrement((PLONG)&x_packet_allocated);
410 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
411 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
412 #else
413 nbl = NdisAllocateNetBufferList(xi->rx_nbl_pool, 0, 0);
414 if (!nbl) {
415 /* buffers will be freed in MakePackets */
416 FUNCTION_MSG("No free nbls\n");
417 //FUNCTION_EXIT();
418 return FALSE;
419 }
421 packet = NdisAllocateNetBuffer(xi->rx_packet_pool, NULL, 0, 0);
422 if (!packet) {
423 FUNCTION_MSG("No free packets\n");
424 NdisFreeNetBufferList(nbl);
425 //FUNCTION_EXIT();
426 return FALSE;
427 }
428 #endif
430 if (!pi->first_mdl->Next && !pi->split_required) {
431 /* a single buffer <= MTU */
432 header_buf = NULL;
433 XenNet_BuildHeader(pi, pi->first_mdl_virtual, pi->first_mdl_length);
434 #if NTDDI_VERSION < NTDDI_VISTA
435 NdisChainBufferAtBack(packet, pi->first_mdl);
436 PACKET_FIRST_PB(packet) = pi->first_pb;
437 #else
438 NET_BUFFER_FIRST_MDL(packet) = pi->first_mdl;
439 NET_BUFFER_CURRENT_MDL(packet) = pi->first_mdl;
440 NET_BUFFER_CURRENT_MDL_OFFSET(packet) = 0;
441 NET_BUFFER_DATA_OFFSET(packet) = 0;
442 NET_BUFFER_DATA_LENGTH(packet) = pi->total_length;
443 NB_FIRST_PB(packet) = pi->first_pb;
444 #endif
445 ref_pb(xi, pi->first_pb);
446 } else {
447 XN_ASSERT(ndis_os_minor_version >= 1);
448 header_buf = get_hb_from_freelist(xi);
449 if (!header_buf) {
450 FUNCTION_MSG("No free header buffers\n");
451 #if NTDDI_VERSION < NTDDI_VISTA
452 NdisUnchainBufferAtFront(packet, &curr_mdl);
453 NdisFreePacket(packet);
454 InterlockedIncrement((PLONG)&x_packet_freed);
455 #else
456 NdisFreeNetBufferList(nbl);
457 NdisFreeNetBuffer(packet);
458 #endif
459 return FALSE;
460 }
461 header_va = (PUCHAR)(header_buf + 1);
462 NdisMoveMemory(header_va, pi->header, pi->header_length);
463 //if (pi->ip_proto == 50) {
464 // FUNCTION_MSG("header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead);
465 // FUNCTION_MSG("ip4_header_length = %d\n", pi->ip4_header_length);
466 // FUNCTION_MSG("tcp_header_length = %d\n", pi->tcp_header_length);
467 //}
468 /* make sure only the header is in the first buffer (or the entire packet, but that is done in the above case) */
469 XenNet_BuildHeader(pi, header_va, MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
470 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
471 XN_ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
472 header_buf->mdl->ByteCount = pi->header_length;
473 mdl_head = mdl_tail = curr_mdl = header_buf->mdl;
474 #if NTDDI_VERSION < NTDDI_VISTA
475 PACKET_FIRST_PB(packet) = header_buf;
476 header_buf->next = pi->curr_pb;
477 NdisChainBufferAtBack(packet, mdl_head);
478 #else
479 NB_FIRST_PB(packet) = header_buf;
480 header_buf->next = pi->curr_pb;
481 NET_BUFFER_FIRST_MDL(packet) = mdl_head;
482 NET_BUFFER_CURRENT_MDL(packet) = mdl_head;
483 NET_BUFFER_CURRENT_MDL_OFFSET(packet) = 0;
484 NET_BUFFER_DATA_OFFSET(packet) = 0;
485 NET_BUFFER_DATA_LENGTH(packet) = pi->header_length;
486 #endif
488 if (pi->split_required) {
489 /* must be ip4 */
490 ULONG tcp_length;
491 USHORT new_ip4_length;
492 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
493 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
494 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
495 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
496 pi->tcp_seq += tcp_length;
497 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
498 /* part of the packet is already present in the header buffer for lookahead */
499 out_remaining = tcp_length - header_extra;
500 XN_ASSERT((LONG)out_remaining >= 0);
501 } else {
502 out_remaining = pi->total_length - pi->header_length;
503 XN_ASSERT((LONG)out_remaining >= 0);
504 }
506 while (out_remaining != 0) {
507 //ULONG in_buffer_offset;
508 ULONG in_buffer_length;
509 ULONG out_length;
511 //if (pi->ip_proto == 50) {
512 // FUNCTION_MSG("in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb);
513 //}
514 if (!pi->curr_mdl || !pi->curr_pb) {
515 FUNCTION_MSG("out of buffers for packet\n");
516 //KdPrint((__DRIVER_NAME " out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb));
517 // TODO: free some stuff or we'll leak
518 /* unchain buffers then free packet */
519 //FUNCTION_EXIT();
520 return FALSE;
521 }
523 in_buffer_length = MmGetMdlByteCount(pi->curr_mdl);
524 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
525 curr_mdl = IoAllocateMdl((PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length, FALSE, FALSE, NULL);
526 XN_ASSERT(curr_mdl);
527 IoBuildPartialMdl(pi->curr_mdl, curr_mdl, (PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length);
528 mdl_tail->Next = curr_mdl;
529 mdl_tail = curr_mdl;
530 curr_mdl->Next = NULL; /* I think this might be redundant */
531 #if NTDDI_VERSION < NTDDI_VISTA
532 #else
533 NET_BUFFER_DATA_LENGTH(packet) += out_length;
534 #endif
535 ref_pb(xi, pi->curr_pb);
536 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
537 if (pi->curr_mdl_offset == in_buffer_length) {
538 pi->curr_mdl = pi->curr_mdl->Next;
539 pi->curr_pb = pi->curr_pb->next;
540 pi->curr_mdl_offset = 0;
541 }
542 out_remaining -= out_length;
543 }
544 #if NTDDI_VERSION < NTDDI_VISTA
545 if (pi->split_required) {
546 // TODO: only if Ip checksum is disabled...
547 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
548 }
549 #endif
550 if (header_extra > 0)
551 pi->header_length -= header_extra;
552 }
554 rc->packet_count++;
555 #if NTDDI_VERSION < NTDDI_VISTA
556 #else
557 NET_BUFFER_LIST_FIRST_NB(nbl) = packet;
558 #endif
560 if (pi->parse_result == PARSE_OK) {
561 #if NTDDI_VERSION < NTDDI_VISTA
562 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
563 packet, TcpIpChecksumPacketInfo);
564 XN_ASSERT(csum_info->Value == 0);
565 if (pi->csum_blank || pi->data_validated || pi->split_required) {
566 BOOLEAN checksum_offload = FALSE;
567 /* we know this is IPv4, and we know Linux always validates the IPv4 checksum for us */
568 if (xi->setting_csum.V4Receive.IpChecksum) {
569 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported) {
570 if (XenNet_CheckIpHeaderSum(pi->header, pi->ip4_header_length))
571 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
572 else
573 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
574 }
575 }
576 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6) {
577 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported) {
578 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
579 checksum_offload = TRUE;
580 }
581 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17) {
582 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
583 checksum_offload = TRUE;
584 }
585 if (pi->csum_blank && (!xi->config_csum_rx_dont_fix || !checksum_offload)) {
586 XenNet_SumPacketData(pi, packet, TRUE);
587 }
588 } else if (xi->config_csum_rx_check && pi->ip_version == 4) {
589 if (xi->setting_csum.V4Receive.IpChecksum) {
590 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported) {
591 if (XenNet_CheckIpHeaderSum(pi->header, pi->ip4_header_length))
592 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
593 else
594 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
595 }
596 }
597 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6) {
598 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported) {
599 if (XenNet_SumPacketData(pi, packet, FALSE)) {
600 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
601 } else {
602 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
603 }
604 }
605 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17) {
606 if (XenNet_SumPacketData(pi, packet, FALSE)) {
607 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
608 } else {
609 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
610 }
611 }
612 }
613 #else
614 csum_info.Value = 0;
615 if (pi->csum_blank || pi->data_validated || pi->mss) {
616 if (pi->ip_proto == 6) {
617 csum_info.Receive.IpChecksumSucceeded = TRUE;
618 csum_info.Receive.TcpChecksumSucceeded = TRUE;
619 } else if (pi->ip_proto == 17) {
620 csum_info.Receive.IpChecksumSucceeded = TRUE;
621 csum_info.Receive.UdpChecksumSucceeded = TRUE;
622 }
623 }
624 NET_BUFFER_LIST_INFO(nbl, TcpIpChecksumNetBufferListInfo) = csum_info.Value;
625 #endif
626 }
628 #if NTDDI_VERSION < NTDDI_VISTA
629 if (!rc->first_packet) {
630 rc->first_packet = packet;
631 } else {
632 PACKET_NEXT_PACKET(rc->last_packet) = packet;
633 }
634 rc->last_packet = packet;
635 rc->packet_count++;
636 #else
637 if (!rc->first_nbl) {
638 rc->first_nbl = nbl;
639 } else {
640 NET_BUFFER_LIST_NEXT_NBL(rc->last_nbl) = nbl;
641 }
642 rc->last_nbl = nbl;
643 NET_BUFFER_LIST_NEXT_NBL(nbl) = NULL;
644 rc->nbl_count++;
645 if (pi->is_multicast) {
646 /* multicast */
647 xi->stats.ifHCInMulticastPkts++;
648 xi->stats.ifHCInMulticastOctets += NET_BUFFER_DATA_LENGTH(packet);
649 } else if (pi->is_broadcast) {
650 /* broadcast */
651 xi->stats.ifHCInBroadcastPkts++;
652 xi->stats.ifHCInBroadcastOctets += NET_BUFFER_DATA_LENGTH(packet);
653 } else {
654 /* unicast */
655 xi->stats.ifHCInUcastPkts++;
656 xi->stats.ifHCInUcastOctets += NET_BUFFER_DATA_LENGTH(packet);
657 }
658 #endif
660 outstanding = InterlockedIncrement(&xi->rx_outstanding);
661 #if NTDDI_VERSION < NTDDI_VISTA
662 if (outstanding > RX_PACKET_HIGH_WATER_MARK || !xi->rx_pb_free) {
663 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
664 InterlockedIncrement((PLONG)&x_packet_resources);
665 } else {
666 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
667 }
668 #if 0
669 /* windows gets lazy about ack packets and holds on to them forever under high load situations. we don't like this */
670 NdisQueryPacketLength(packet, &packet_length);
671 if (pi->parse_result != PARSE_OK || (pi->ip_proto == 6 && packet_length <= NDIS_STATUS_RESOURCES_MAX_LENGTH))
672 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
673 else
674 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
675 #endif
676 #endif
678 //FUNCTION_EXIT();
679 return TRUE;
680 }
682 static VOID
683 XenNet_MakePackets(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi)
684 {
685 UCHAR psh;
686 shared_buffer_t *page_buf;
688 XenNet_ParsePacketHeader(pi, NULL, XN_HDR_SIZE + xi->current_lookahead);
690 if (!XenNet_FilterAcceptPacket(xi, pi)) {
691 goto done;
692 }
694 if (pi->split_required) {
695 #if NTDDI_VERSION < NTDDI_VISTA
696 /* need to split to mss for NDIS5 */
697 #else
698 switch (xi->current_gso_rx_split_type) {
699 case RX_LSO_SPLIT_HALF:
700 pi->mss = max((pi->tcp_length + 1) / 2, pi->mss);
701 break;
702 case RX_LSO_SPLIT_NONE:
703 pi->mss = 65535;
704 break;
705 }
706 #endif
707 }
709 switch (pi->ip_proto) {
710 case 6: // TCP
711 if (pi->split_required)
712 break;
713 /* fall through */
714 case 17: // UDP
715 if (!XenNet_MakePacket(xi, rc, pi)) {
716 FUNCTION_MSG("Failed to make packet\n");
717 #if NTDDI_VERSION < NTDDI_VISTA
718 xi->stat_rx_no_buffer++;
719 #else
720 xi->stats.ifInDiscards++;
721 #endif
722 goto done;
723 }
724 goto done;
725 default:
726 if (!XenNet_MakePacket(xi, rc, pi)) {
727 FUNCTION_MSG("Failed to make packet\n");
728 #if NTDDI_VERSION < NTDDI_VISTA
729 xi->stat_rx_no_buffer++;
730 #else
731 xi->stats.ifInDiscards++;
732 #endif
733 goto done;
734 }
735 goto done;
736 }
738 /* this is the split_required code */
739 pi->tcp_remaining = pi->tcp_length;
741 /* we can make certain assumptions here as the following code is only for tcp4 */
742 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
743 while (pi->tcp_remaining) {
744 if (!XenNet_MakePacket(xi, rc, pi)) {
745 FUNCTION_MSG("Failed to make packet\n");
746 #if NTDDI_VERSION < NTDDI_VISTA
747 xi->stat_rx_no_buffer++;
748 #else
749 xi->stats.ifInDiscards++;
750 #endif
751 break; /* we are out of memory - just drop the packets */
752 }
753 if (psh) {
754 if (pi->tcp_remaining)
755 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
756 else
757 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
758 }
759 }
760 done:
761 page_buf = pi->first_pb;
762 while (page_buf) {
763 shared_buffer_t *next_pb = page_buf->next;
764 put_pb_on_freelist(xi, page_buf); /* this doesn't actually free the page_puf if there are outstanding references */
765 page_buf = next_pb;
766 }
767 XenNet_ClearPacketInfo(pi);
768 //FUNCTION_EXIT();
769 return;
770 }
772 #if NTDDI_VERSION < NTDDI_VISTA
773 /* called at DISPATCH_LEVEL */
774 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
775 VOID
776 XenNet_ReturnPacket(NDIS_HANDLE adapter_context, PNDIS_PACKET packet) {
777 struct xennet_info *xi = adapter_context;
778 PNDIS_BUFFER buffer;
779 shared_buffer_t *page_buf = PACKET_FIRST_PB(packet);
781 #ifdef RX_STATS
782 {
783 UINT packet_length;
784 UINT buffer_count;
786 NdisQueryPacket(packet, NULL, &buffer_count, NULL, &packet_length);
787 if (packet_length <= 64) {
788 InterlockedIncrement((PLONG)&x_0_64_packet_returned);
789 } else if (packet_length <= 128) {
790 InterlockedIncrement((PLONG)&x_65_128_packet_returned);
791 } else if (packet_length <= 256) {
792 InterlockedIncrement((PLONG)&x_129_256_packet_returned);
793 } else if (packet_length <= 512) {
794 InterlockedIncrement((PLONG)&x_257_512_packet_returned);
795 } else if (packet_length <= 1024) {
796 InterlockedIncrement((PLONG)&x_513_1024_packet_returned);
797 } else if (packet_length <= 1516) {
798 InterlockedIncrement((PLONG)&x_1025_1514_packet_returned);
799 } else {
800 InterlockedIncrement((PLONG)&x_1515_65535_packet_returned);
801 }
802 if (buffer_count == 1) {
803 InterlockedIncrement((PLONG)&x_1_frag_packet_returned);
804 } else if (buffer_count == 2) {
805 InterlockedIncrement((PLONG)&x_2_frag_packet_returned);
806 } else if (buffer_count == 3) {
807 InterlockedIncrement((PLONG)&x_3_frag_packet_returned);
808 } else {
809 InterlockedIncrement((PLONG)&x_4_or_more_frag_packet_returned);
810 }
811 }
812 #endif
814 //FUNCTION_ENTER();
815 NdisUnchainBufferAtFront(packet, &buffer);
817 while (buffer) {
818 shared_buffer_t *next_buf;
819 XN_ASSERT(page_buf);
820 next_buf = page_buf->next;
821 if (!page_buf->virtual) {
822 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
823 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(buffer) - 1);
824 } else {
825 if (buffer != page_buf->mdl)
826 NdisFreeBuffer(buffer);
827 put_pb_on_freelist(xi, page_buf);
828 }
829 NdisUnchainBufferAtFront(packet, &buffer);
830 page_buf = next_buf;
831 }
833 NdisFreePacket(packet);
834 InterlockedIncrement((PLONG)&x_packet_freed);
835 InterlockedDecrement(&xi->rx_outstanding);
836 if (!xi->rx_outstanding && xi->device_state != DEVICE_STATE_ACTIVE)
837 KeSetEvent(&xi->rx_idle_event, IO_NO_INCREMENT, FALSE);
838 //FUNCTION_EXIT();
839 }
840 #else
841 /* called at <= DISPATCH_LEVEL */
842 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
843 VOID
844 XenNet_ReturnNetBufferLists(NDIS_HANDLE adapter_context, PNET_BUFFER_LIST curr_nbl, ULONG return_flags)
845 {
846 struct xennet_info *xi = adapter_context;
847 UNREFERENCED_PARAMETER(return_flags);
849 //FUNCTION_ENTER();
851 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
853 XN_ASSERT(xi);
854 while (curr_nbl)
855 {
856 PNET_BUFFER_LIST next_nbl;
857 PNET_BUFFER curr_nb;
859 next_nbl = NET_BUFFER_LIST_NEXT_NBL(curr_nbl);
860 curr_nb = NET_BUFFER_LIST_FIRST_NB(curr_nbl);
861 while (curr_nb)
862 {
863 PNET_BUFFER next_nb;
864 PMDL curr_mdl;
865 shared_buffer_t *page_buf;
867 next_nb = NET_BUFFER_NEXT_NB(curr_nb);
868 curr_mdl = NET_BUFFER_FIRST_MDL(curr_nb);
869 page_buf = NB_FIRST_PB(curr_nb);
870 while (curr_mdl)
871 {
872 shared_buffer_t *next_buf;
873 PMDL next_mdl;
875 XN_ASSERT(page_buf); /* make sure that there is a pb to match this mdl */
876 next_mdl = curr_mdl->Next;
877 next_buf = page_buf->next;
878 if (!page_buf->virtual)
879 {
880 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
881 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(curr_mdl) - 1);
882 }
883 else
884 {
885 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
886 if (curr_mdl != page_buf->mdl)
887 {
888 //KdPrint((__DRIVER_NAME " curr_mdl = %p, page_buf->mdl = %p\n", curr_mdl, page_buf->mdl));
889 IoFreeMdl(curr_mdl);
890 }
891 put_pb_on_freelist(xi, page_buf);
892 }
893 curr_mdl = next_mdl;
894 page_buf = next_buf;
895 }
897 NdisFreeNetBuffer(curr_nb);
898 InterlockedDecrement(&xi->rx_outstanding);
900 curr_nb = next_nb;
901 }
902 NdisFreeNetBufferList(curr_nbl);
903 curr_nbl = next_nbl;
904 }
906 if (!xi->rx_outstanding && xi->device_state != DEVICE_STATE_ACTIVE)
907 KeSetEvent(&xi->rx_idle_event, IO_NO_INCREMENT, FALSE);
909 //FUNCTION_EXIT();
910 }
911 #endif
913 /* We limit the number of packets per interrupt so that acks get a chance
914 under high rx load. The DPC is immediately re-scheduled */
916 #define MAXIMUM_PACKETS_PER_INDICATE 32
918 #define MAXIMUM_PACKETS_PER_INTERRUPT 2560 /* this is calculated before large packet split */
919 #define MAXIMUM_DATA_PER_INTERRUPT (MAXIMUM_PACKETS_PER_INTERRUPT * 1500) /* help account for large packets */
921 // Called at DISPATCH_LEVEL
922 BOOLEAN
923 XenNet_RxBufferCheck(struct xennet_info *xi)
924 {
925 RING_IDX cons, prod;
926 ULONG packet_count = 0;
927 ULONG packet_data = 0;
928 ULONG buffer_count = 0;
929 USHORT id;
930 int more_to_do = FALSE;
931 shared_buffer_t *page_buf;
932 #if NTDDI_VERSION < NTDDI_VISTA
933 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
934 PNDIS_PACKET first_header_only_packet;
935 PNDIS_PACKET last_header_only_packet;
936 #else
937 #endif
938 //ULONG nbl_count = 0;
939 ULONG interim_packet_data = 0;
940 struct netif_extra_info *ei;
941 rx_context_t rc;
942 packet_info_t *pi = &xi->rxpi[KeGetCurrentProcessorNumber() & 0xff];
943 shared_buffer_t *head_buf = NULL;
944 shared_buffer_t *tail_buf = NULL;
945 shared_buffer_t *last_buf = NULL;
946 BOOLEAN extra_info_flag = FALSE;
947 BOOLEAN more_data_flag = FALSE;
948 BOOLEAN dont_set_event;
949 //FUNCTION_ENTER();
951 #if NTDDI_VERSION < NTDDI_VISTA
952 rc.first_packet = NULL;
953 rc.last_packet = NULL;
954 rc.packet_count = 0;
955 #else
956 rc.first_nbl = NULL;
957 rc.last_nbl = NULL;
958 rc.packet_count = 0;
959 rc.nbl_count = 0;
960 #endif
962 /* get all the buffers off the ring as quickly as possible so the lock is held for a minimum amount of time */
963 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
965 if (xi->device_state != DEVICE_STATE_ACTIVE) {
966 /* there is a chance that our Dpc had been queued just before the shutdown... */
967 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
968 return FALSE;
969 }
971 if (xi->rx_partial_buf) {
972 head_buf = xi->rx_partial_buf;
973 tail_buf = xi->rx_partial_buf;
974 while (tail_buf->next)
975 tail_buf = tail_buf->next;
976 more_data_flag = xi->rx_partial_more_data_flag;
977 extra_info_flag = xi->rx_partial_extra_info_flag;
978 xi->rx_partial_buf = NULL;
979 }
981 do {
982 prod = xi->rx_ring.sring->rsp_prod;
983 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
985 for (cons = xi->rx_ring.rsp_cons; cons != prod && packet_count < MAXIMUM_PACKETS_PER_INTERRUPT && packet_data < MAXIMUM_DATA_PER_INTERRUPT; cons++) {
986 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
987 page_buf = xi->rx_ring_pbs[id];
988 XN_ASSERT(page_buf);
989 xi->rx_ring_pbs[id] = NULL;
990 xi->rx_id_free++;
991 memcpy(&page_buf->rsp, RING_GET_RESPONSE(&xi->rx_ring, cons), max(sizeof(struct netif_rx_response), sizeof(struct netif_extra_info)));
992 if (!extra_info_flag) {
993 if (page_buf->rsp.status <= 0 || page_buf->rsp.offset + page_buf->rsp.status > PAGE_SIZE) {
994 FUNCTION_MSG("Error: rsp offset %d, size %d\n",
995 page_buf->rsp.offset, page_buf->rsp.status);
996 XN_ASSERT(!extra_info_flag);
997 put_pb_on_freelist(xi, page_buf);
998 continue;
999 }
1002 if (!head_buf) {
1003 head_buf = page_buf;
1004 tail_buf = page_buf;
1005 } else {
1006 tail_buf->next = page_buf;
1007 tail_buf = page_buf;
1009 page_buf->next = NULL;
1011 if (extra_info_flag) {
1012 ei = (struct netif_extra_info *)&page_buf->rsp;
1013 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
1014 } else {
1015 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
1016 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
1017 interim_packet_data += page_buf->rsp.status;
1020 if (!extra_info_flag && !more_data_flag) {
1021 last_buf = page_buf;
1022 packet_count++;
1023 packet_data += interim_packet_data;
1024 interim_packet_data = 0;
1026 buffer_count++;
1028 xi->rx_ring.rsp_cons = cons;
1030 /* Give netback more buffers */
1031 XenNet_FillRing(xi);
1033 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
1034 break;
1036 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx_ring);
1037 if (!more_to_do) {
1038 xi->rx_ring.sring->rsp_event = xi->rx_ring.rsp_cons + 1;
1039 KeMemoryBarrier();
1040 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx_ring);
1042 } while (more_to_do);
1044 /* anything past last_buf belongs to an incomplete packet... */
1045 if (last_buf && last_buf->next)
1047 FUNCTION_MSG("Partial receive\n");
1048 xi->rx_partial_buf = last_buf->next;
1049 xi->rx_partial_more_data_flag = more_data_flag;
1050 xi->rx_partial_extra_info_flag = extra_info_flag;
1051 last_buf->next = NULL;
1054 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
1056 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
1058 /* fire again immediately */
1059 FUNCTION_MSG("Dpc Duration Exceeded\n");
1060 /* we want the Dpc on the end of the queue. By definition we are already on the right CPU so we know the Dpc queue will be run immediately */
1061 // KeSetImportanceDpc(&xi->rxtx_dpc, MediumImportance);
1062 KeInsertQueueDpc(&xi->rxtx_dpc, NULL, NULL);
1063 /* dont set an event in TX path */
1064 dont_set_event = TRUE;
1066 else
1068 /* make sure the Dpc queue is run immediately next interrupt */
1069 // KeSetImportanceDpc(&xi->rxtx_dpc, HighImportance);
1070 /* set an event in TX path */
1071 dont_set_event = FALSE;
1074 /* make packets out of the buffers */
1075 page_buf = head_buf;
1076 extra_info_flag = FALSE;
1077 more_data_flag = FALSE;
1079 while (page_buf) {
1080 shared_buffer_t *next_buf = page_buf->next;
1081 PMDL mdl;
1083 page_buf->next = NULL;
1084 if (extra_info_flag) {
1085 //KdPrint((__DRIVER_NAME " processing extra info\n"));
1086 ei = (struct netif_extra_info *)&page_buf->rsp;
1087 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
1088 switch (ei->type)
1090 case XEN_NETIF_EXTRA_TYPE_GSO:
1091 switch (ei->u.gso.type) {
1092 case XEN_NETIF_GSO_TYPE_TCPV4:
1093 pi->mss = ei->u.gso.size;
1094 // TODO - put this assertion somewhere XN_ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
1095 break;
1096 default:
1097 FUNCTION_MSG("Unknown GSO type (%d) detected\n", ei->u.gso.type);
1098 break;
1100 break;
1101 default:
1102 FUNCTION_MSG("Unknown extra info type (%d) detected\n", ei->type);
1103 break;
1105 put_pb_on_freelist(xi, page_buf);
1106 } else {
1107 XN_ASSERT(!page_buf->rsp.offset);
1108 if (!more_data_flag) { // handling the packet's 1st buffer
1109 if (page_buf->rsp.flags & NETRXF_csum_blank)
1110 pi->csum_blank = TRUE;
1111 if (page_buf->rsp.flags & NETRXF_data_validated)
1112 pi->data_validated = TRUE;
1114 mdl = page_buf->mdl;
1115 mdl->ByteCount = page_buf->rsp.status; //NdisAdjustBufferLength(mdl, page_buf->rsp.status);
1116 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
1117 if (pi->first_pb) {
1118 XN_ASSERT(pi->curr_pb);
1119 //KdPrint((__DRIVER_NAME " additional buffer\n"));
1120 pi->curr_pb->next = page_buf;
1121 pi->curr_pb = page_buf;
1122 XN_ASSERT(pi->curr_mdl);
1123 pi->curr_mdl->Next = mdl;
1124 pi->curr_mdl = mdl;
1125 } else {
1126 pi->first_pb = page_buf;
1127 pi->curr_pb = page_buf;
1128 pi->first_mdl = mdl;
1129 pi->curr_mdl = mdl;
1131 //pi->mdl_count++;
1132 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
1133 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
1134 pi->total_length = pi->total_length + page_buf->rsp.status;
1137 /* Packet done, add it to the list */
1138 if (!more_data_flag && !extra_info_flag) {
1139 pi->curr_pb = pi->first_pb;
1140 pi->curr_mdl = pi->first_mdl;
1141 XenNet_MakePackets(xi, &rc, pi);
1144 page_buf = next_buf;
1146 XN_ASSERT(!more_data_flag && !extra_info_flag);
1148 #if NTDDI_VERSION < NTDDI_VISTA
1149 packet_count = 0;
1150 first_header_only_packet = NULL;
1151 last_header_only_packet = NULL;
1153 while (rc.first_packet) {
1154 PNDIS_PACKET packet;
1155 NDIS_STATUS status;
1156 UINT packet_length;
1157 UINT buffer_count;
1159 packet = rc.first_packet;
1160 XN_ASSERT(PACKET_FIRST_PB(packet));
1161 rc.first_packet = PACKET_NEXT_PACKET(packet);
1162 status = NDIS_GET_PACKET_STATUS(packet);
1163 if (status == NDIS_STATUS_RESOURCES) {
1164 if (!first_header_only_packet) {
1165 first_header_only_packet = packet;
1166 } else {
1167 PACKET_NEXT_PACKET(last_header_only_packet) = packet;
1169 last_header_only_packet = packet;
1170 PACKET_NEXT_PACKET(packet) = NULL;
1172 NdisQueryPacket(packet, NULL, &buffer_count, NULL, &packet_length);
1173 if (packet_length <= 64) {
1174 InterlockedIncrement((PLONG)&x_0_64_packet_indicated);
1175 } else if (packet_length <= 128) {
1176 InterlockedIncrement((PLONG)&x_65_128_packet_indicated);
1177 } else if (packet_length <= 256) {
1178 InterlockedIncrement((PLONG)&x_129_256_packet_indicated);
1179 } else if (packet_length <= 512) {
1180 InterlockedIncrement((PLONG)&x_257_512_packet_indicated);
1181 } else if (packet_length <= 1024) {
1182 InterlockedIncrement((PLONG)&x_513_1024_packet_indicated);
1183 } else if (packet_length <= 1516) {
1184 InterlockedIncrement((PLONG)&x_1025_1514_packet_indicated);
1185 } else {
1186 InterlockedIncrement((PLONG)&x_1515_65535_packet_indicated);
1188 if (buffer_count == 1) {
1189 InterlockedIncrement((PLONG)&x_1_frag_packet_indicated);
1190 } else if (buffer_count == 2) {
1191 InterlockedIncrement((PLONG)&x_2_frag_packet_indicated);
1192 } else if (buffer_count == 3) {
1193 InterlockedIncrement((PLONG)&x_3_frag_packet_indicated);
1194 } else {
1195 InterlockedIncrement((PLONG)&x_4_or_more_frag_packet_indicated);
1197 packets[packet_count++] = packet;
1198 /* if we indicate a packet with NDIS_STATUS_RESOURCES then any following packet can't be NDIS_STATUS_SUCCESS */
1199 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || !rc.first_packet
1200 || (NDIS_GET_PACKET_STATUS(rc.first_packet) == NDIS_STATUS_SUCCESS
1201 && status == NDIS_STATUS_RESOURCES)) {
1202 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
1203 packet_count = 0;
1206 /* now return the packets for which we indicated NDIS_STATUS_RESOURCES */
1207 while (first_header_only_packet) {
1208 PNDIS_PACKET packet = first_header_only_packet;
1209 first_header_only_packet = PACKET_NEXT_PACKET(packet);
1210 XenNet_ReturnPacket(xi, packet);
1213 LARGE_INTEGER current_time;
1214 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
1215 KeQuerySystemTime(&current_time);
1216 if (current_time.QuadPart >= x_next_print_time.QuadPart) {
1217 dump_x_stats();
1218 x_next_print_time.QuadPart = current_time.QuadPart + 10 * 1000 * 1000 * 10; /* 10 seconds */
1220 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
1222 #else
1223 if (rc.first_nbl) {
1224 NdisMIndicateReceiveNetBufferLists(xi->adapter_handle, rc.first_nbl,
1225 NDIS_DEFAULT_PORT_NUMBER, rc.nbl_count,
1226 NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL
1227 //| NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE
1228 | NDIS_RECEIVE_FLAGS_PERFECT_FILTERED);
1230 #endif
1231 //FUNCTION_EXIT();
1232 return dont_set_event;
1235 static VOID
1236 XenNet_BufferFree(xennet_info_t *xi)
1238 shared_buffer_t *sb;
1239 int i;
1241 for (i = 0; i < NET_RX_RING_SIZE; i++) {
1242 if (xi->rx_ring_pbs[i] != NULL) {
1243 put_pb_on_freelist(xi, xi->rx_ring_pbs[i]);
1244 xi->rx_ring_pbs[i] = NULL;
1248 /* because we are shutting down this won't allocate new ones */
1249 while ((sb = get_pb_from_freelist(xi)) != NULL) {
1250 XnEndAccess(xi->handle,
1251 sb->gref, FALSE, (ULONG)'XNRX');
1252 IoFreeMdl(sb->mdl);
1253 ExFreePoolWithTag(sb->virtual, XENNET_POOL_TAG);
1254 ExFreePoolWithTag(sb, XENNET_POOL_TAG);
1256 while ((sb = get_hb_from_freelist(xi)) != NULL) {
1257 IoFreeMdl(sb->mdl);
1258 ExFreePoolWithTag(sb, XENNET_POOL_TAG);
1262 BOOLEAN
1263 XenNet_RxInit(xennet_info_t *xi) {
1264 #if NTDDI_VERSION < NTDDI_VISTA
1265 NDIS_STATUS status;
1266 #else
1267 NET_BUFFER_LIST_POOL_PARAMETERS nbl_pool_parameters;
1268 NET_BUFFER_POOL_PARAMETERS nb_pool_parameters;
1269 #endif
1270 int ret;
1271 int i;
1273 FUNCTION_ENTER();
1275 // this stuff needs to be done once only...
1276 KeInitializeSpinLock(&xi->rx_lock);
1277 KeInitializeEvent(&xi->rx_idle_event, SynchronizationEvent, FALSE);
1278 xi->rxpi = ExAllocatePoolWithTagPriority(NonPagedPool, sizeof(packet_info_t) * NdisSystemProcessorCount(), XENNET_POOL_TAG, NormalPoolPriority);
1279 if (!xi->rxpi) {
1280 FUNCTION_MSG("ExAllocatePoolWithTagPriority failed\n");
1281 return FALSE;
1283 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount());
1285 ret = stack_new(&xi->rx_pb_stack, NET_RX_RING_SIZE * 4);
1286 if (!ret) {
1287 FUNCTION_MSG("Failed to allocate rx_pb_stack\n");
1288 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1289 return FALSE;
1291 ret = stack_new(&xi->rx_hb_stack, NET_RX_RING_SIZE * 4);
1292 if (!ret) {
1293 FUNCTION_MSG("Failed to allocate rx_hb_stack\n");
1294 stack_delete(xi->rx_pb_stack, NULL, NULL);
1295 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1296 return FALSE;
1299 xi->rx_id_free = NET_RX_RING_SIZE;
1300 xi->rx_outstanding = 0;
1302 for (i = 0; i < NET_RX_RING_SIZE; i++) {
1303 xi->rx_ring_pbs[i] = NULL;
1306 #if NTDDI_VERSION < NTDDI_VISTA
1307 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4, PROTOCOL_RESERVED_SIZE_IN_PACKET);
1308 if (status != NDIS_STATUS_SUCCESS) {
1309 FUNCTION_MSG("NdisAllocatePacketPool failed with 0x%x\n", status);
1310 return FALSE;
1312 #else
1313 nbl_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
1314 nbl_pool_parameters.Header.Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
1315 nbl_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
1316 nbl_pool_parameters.ProtocolId = NDIS_PROTOCOL_ID_DEFAULT;
1317 nbl_pool_parameters.fAllocateNetBuffer = FALSE;
1318 nbl_pool_parameters.ContextSize = 0;
1319 nbl_pool_parameters.PoolTag = XENNET_POOL_TAG;
1320 nbl_pool_parameters.DataSize = 0; /* NET_BUFFERS are always allocated separately */
1322 xi->rx_nbl_pool = NdisAllocateNetBufferListPool(xi->adapter_handle, &nbl_pool_parameters);
1323 if (!xi->rx_nbl_pool) {
1324 FUNCTION_MSG("NdisAllocateNetBufferListPool failed\n");
1325 return FALSE;
1328 nb_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
1329 nb_pool_parameters.Header.Revision = NET_BUFFER_POOL_PARAMETERS_REVISION_1;
1330 nb_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_POOL_PARAMETERS_REVISION_1;
1331 nb_pool_parameters.PoolTag = XENNET_POOL_TAG;
1332 nb_pool_parameters.DataSize = 0; /* the buffers come from the ring */
1333 xi->rx_packet_pool = NdisAllocateNetBufferPool(xi->adapter_handle, &nb_pool_parameters);
1334 if (!xi->rx_packet_pool) {
1335 FUNCTION_MSG("NdisAllocateNetBufferPool (rx_packet_pool) failed\n");
1336 return FALSE;
1338 #endif
1339 XenNet_FillRing(xi);
1341 FUNCTION_EXIT();
1343 return TRUE;
1346 VOID
1347 XenNet_RxShutdown(xennet_info_t *xi) {
1348 KIRQL old_irql;
1349 UNREFERENCED_PARAMETER(xi);
1351 FUNCTION_ENTER();
1353 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1354 while (xi->rx_outstanding) {
1355 FUNCTION_MSG("Waiting for %d packets to be returned\n", xi->rx_outstanding);
1356 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1357 KeWaitForSingleObject(&xi->rx_idle_event, Executive, KernelMode, FALSE, NULL);
1358 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1360 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1362 XenNet_BufferFree(xi);
1364 stack_delete(xi->rx_pb_stack, NULL, NULL);
1365 stack_delete(xi->rx_hb_stack, NULL, NULL);
1368 ExFreePoolWithTag(xi->rxpi, XENNET_POOL_TAG);
1370 #if NTDDI_VERSION < NTDDI_VISTA
1371 NdisFreePacketPool(xi->rx_packet_pool);
1372 #else
1373 NdisFreeNetBufferPool(xi->rx_packet_pool);
1374 NdisFreeNetBufferListPool(xi->rx_nbl_pool);
1375 #endif
1377 FUNCTION_EXIT();
1378 return;