win-pvdrivers

view xennet/xennet_rx.c @ 900:938de6b8623d

fix memory leak on suspend/resume
author James Harper <james.harper@bendigoit.com.au>
date Thu Mar 31 20:20:36 2011 +1100 (2011-03-31)
parents 0b3220aea661
children 2e4f05dd4dfe
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 /* Not really necessary but keeps PREfast happy */
24 #if (NTDDI_VERSION >= NTDDI_WINXP)
25 static KDEFERRED_ROUTINE XenNet_RxBufferCheck;
26 #endif
28 LONG rx_pb_outstanding = 0;
30 static __inline shared_buffer_t *
31 get_pb_from_freelist(struct xennet_info *xi)
32 {
33 NDIS_STATUS status;
34 shared_buffer_t *pb;
35 PVOID ptr_ref;
37 if (stack_pop(xi->rx_pb_stack, &ptr_ref))
38 {
39 pb = ptr_ref;
40 pb->ref_count = 1;
41 InterlockedDecrement(&xi->rx_pb_free);
42 InterlockedIncrement(&rx_pb_outstanding);
43 return pb;
44 }
46 /* don't allocate a new one if we are shutting down */
47 if (xi->shutting_down)
48 return NULL;
50 status = NdisAllocateMemoryWithTag(&pb, sizeof(shared_buffer_t), XENNET_POOL_TAG);
51 if (status != STATUS_SUCCESS)
52 {
53 return NULL;
54 }
55 status = NdisAllocateMemoryWithTag(&pb->virtual, PAGE_SIZE, XENNET_POOL_TAG);
56 if (status != STATUS_SUCCESS)
57 {
58 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
59 return NULL;
60 }
61 pb->gref = (grant_ref_t)xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
62 (ULONG)(MmGetPhysicalAddress(pb->virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
63 if (pb->gref == INVALID_GRANT_REF)
64 {
65 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
66 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
67 return NULL;
68 }
69 pb->offset = (USHORT)(ULONG_PTR)pb->virtual & (PAGE_SIZE - 1);
70 NdisAllocateBuffer(&status, &pb->buffer, xi->rx_buffer_pool, (PUCHAR)pb->virtual, PAGE_SIZE);
71 if (status != STATUS_SUCCESS)
72 {
73 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
74 pb->gref, FALSE, (ULONG)'XNRX');
75 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
76 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
77 return NULL;
78 }
79 InterlockedIncrement(&rx_pb_outstanding);
80 pb->ref_count = 1;
81 return pb;
82 }
84 static __inline VOID
85 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
86 {
87 UNREFERENCED_PARAMETER(xi);
88 InterlockedIncrement(&pb->ref_count);
89 }
91 static __inline VOID
92 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
93 {
94 if (InterlockedDecrement(&pb->ref_count) == 0)
95 {
96 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
97 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
98 pb->next = NULL;
99 stack_push(xi->rx_pb_stack, pb);
100 InterlockedIncrement(&xi->rx_pb_free);
101 InterlockedDecrement(&rx_pb_outstanding);
102 }
103 }
105 // Called at DISPATCH_LEVEL with rx lock held
106 static NDIS_STATUS
107 XenNet_FillRing(struct xennet_info *xi)
108 {
109 unsigned short id;
110 shared_buffer_t *page_buf;
111 ULONG i, notify;
112 ULONG batch_target;
113 RING_IDX req_prod = xi->rx.req_prod_pvt;
114 netif_rx_request_t *req;
116 //FUNCTION_ENTER();
118 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
120 if (batch_target < (xi->rx_target >> 2))
121 {
122 //FUNCTION_EXIT();
123 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
124 }
126 for (i = 0; i < batch_target; i++)
127 {
128 page_buf = get_pb_from_freelist(xi);
129 if (!page_buf)
130 {
131 KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
132 break;
133 }
134 xi->rx_id_free--;
136 /* Give to netback */
137 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
138 ASSERT(xi->rx_ring_pbs[id] == NULL);
139 xi->rx_ring_pbs[id] = page_buf;
140 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
141 req->id = id;
142 req->gref = page_buf->gref;
143 ASSERT(req->gref != INVALID_GRANT_REF);
144 }
145 KeMemoryBarrier();
146 xi->rx.req_prod_pvt = req_prod + i;
147 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
148 if (notify)
149 {
150 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
151 }
153 //FUNCTION_EXIT();
155 return NDIS_STATUS_SUCCESS;
156 }
158 LONG total_allocated_packets = 0;
159 LONG dpc_limit_hit = 0;
160 LARGE_INTEGER last_print_time;
162 /* lock free */
163 static PNDIS_PACKET
164 get_packet_from_freelist(struct xennet_info *xi)
165 {
166 NDIS_STATUS status;
167 PNDIS_PACKET packet;
169 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
170 if (status != NDIS_STATUS_SUCCESS)
171 {
172 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
173 return NULL;
174 }
175 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
176 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
177 InterlockedIncrement(&total_allocated_packets);
178 return packet;
179 }
181 /* lock free */
182 static VOID
183 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
184 {
185 LARGE_INTEGER current_time;
187 UNREFERENCED_PARAMETER(xi);
189 InterlockedDecrement(&total_allocated_packets);
190 NdisFreePacket(packet);
191 KeQuerySystemTime(&current_time);
192 if ((int)total_allocated_packets < 0 || (current_time.QuadPart - last_print_time.QuadPart) / 10000 > 1000)
193 {
194 last_print_time.QuadPart = current_time.QuadPart;
195 KdPrint(("total_allocated_packets = %d, rx_outstanding = %d, dpc_limit_hit = %d, rx_pb_outstanding = %d, rx_pb_free = %d\n", total_allocated_packets, xi->rx_outstanding, dpc_limit_hit, rx_pb_outstanding, xi->rx_pb_free));
196 }
197 }
199 static PNDIS_PACKET
200 XenNet_MakePacket(struct xennet_info *xi, packet_info_t *pi)
201 {
202 NDIS_STATUS status;
203 PNDIS_PACKET packet;
204 PNDIS_BUFFER out_buffer;
205 USHORT new_ip4_length;
206 PUCHAR header_va;
207 ULONG out_remaining;
208 ULONG tcp_length;
209 ULONG header_extra;
210 shared_buffer_t *header_buf;
212 //FUNCTION_ENTER();
214 packet = get_packet_from_freelist(xi);
215 if (packet == NULL)
216 {
217 /* buffers will be freed in MakePackets */
218 //KdPrint((__DRIVER_NAME " No free packets\n"));
219 //FUNCTION_EXIT();
220 return NULL;
221 }
223 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
224 if (!header_buf)
225 {
226 KdPrint((__DRIVER_NAME " No free header buffers\n"));
227 put_packet_on_freelist(xi, packet);
228 return NULL;
229 }
230 header_va = (PUCHAR)(header_buf + 1);
231 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
232 NdisMoveMemory(header_va, pi->header, pi->header_length);
233 //KdPrint((__DRIVER_NAME " header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead));
234 //KdPrint((__DRIVER_NAME " ip4_header_length = %d\n", pi->ip4_header_length));
235 //KdPrint((__DRIVER_NAME " tcp_header_length = %d\n", pi->tcp_header_length));
236 /* make sure we satisfy the lookahead requirement */
238 if (pi->split_required)
239 {
240 /* for split packets we need to make sure the 'header' is no bigger than header+mss bytes */
241 XenNet_BuildHeader(pi, header_va, min((ULONG)MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length + pi->mss, MAX_ETH_HEADER_LENGTH + max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead)));
242 }
243 else
244 {
245 XenNet_BuildHeader(pi, header_va, max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead) + MAX_ETH_HEADER_LENGTH);
246 }
247 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
248 ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
249 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
250 if (status != STATUS_SUCCESS)
251 {
252 KdPrint((__DRIVER_NAME " No free header buffers\n"));
253 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, header_buf);
254 put_packet_on_freelist(xi, packet);
255 return NULL;
256 }
257 NdisChainBufferAtBack(packet, out_buffer);
258 *(shared_buffer_t **)&packet->MiniportReservedEx[0] = header_buf;
259 header_buf->next = pi->curr_pb;
261 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too... maybe
263 if (pi->split_required)
264 {
265 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
266 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
267 //KdPrint((__DRIVER_NAME " new_ip4_length = %d\n", new_ip4_length));
268 //KdPrint((__DRIVER_NAME " this tcp_length = %d\n", tcp_length));
269 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
270 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
271 pi->tcp_seq += tcp_length;
272 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
273 /* part of the packet is already present in the header buffer for lookahead */
274 out_remaining = tcp_length - header_extra;
275 ASSERT((LONG)out_remaining >= 0);
276 }
277 else
278 {
279 out_remaining = pi->total_length - pi->header_length;
280 ASSERT((LONG)out_remaining >= 0);
281 }
282 //KdPrint((__DRIVER_NAME " before loop - out_remaining = %d\n", out_remaining));
284 while (out_remaining != 0)
285 {
286 ULONG in_buffer_offset;
287 ULONG in_buffer_length;
288 ULONG out_length;
290 //KdPrint((__DRIVER_NAME " in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_buffer, pi->curr_pb));
291 if (!pi->curr_buffer || !pi->curr_pb)
292 {
293 KdPrint((__DRIVER_NAME " out of buffers for packet\n"));
294 KdPrint((__DRIVER_NAME " out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_buffer, pi->curr_pb));
295 // TODO: free some stuff or we'll leak
296 /* unchain buffers then free packet */
297 return NULL;
298 }
299 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
300 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
301 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
302 ASSERT(status == STATUS_SUCCESS); //TODO: properly handle error
303 NdisChainBufferAtBack(packet, out_buffer);
304 ref_pb(xi, pi->curr_pb);
305 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
306 if (pi->curr_mdl_offset == in_buffer_length)
307 {
308 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
309 pi->curr_pb = pi->curr_pb->next;
310 pi->curr_mdl_offset = 0;
311 }
312 out_remaining -= out_length;
313 }
314 if (pi->split_required)
315 {
316 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
317 }
318 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
319 if (header_extra > 0)
320 pi->header_length -= header_extra;
321 ASSERT(*(shared_buffer_t **)&packet->MiniportReservedEx[0]);
322 //FUNCTION_EXIT();
323 return packet;
324 }
326 /*
327 Windows appears to insist that the checksum on received packets is correct, and won't
328 believe us when we lie about it, which happens when the packet is generated on the
329 same bridge in Dom0. Doh!
330 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
331 */
333 static BOOLEAN
334 XenNet_SumPacketData(
335 packet_info_t *pi,
336 PNDIS_PACKET packet,
337 BOOLEAN set_csum
338 )
339 {
340 USHORT i;
341 PUCHAR buffer;
342 PMDL mdl;
343 UINT total_length;
344 UINT data_length;
345 UINT buffer_length;
346 USHORT buffer_offset;
347 ULONG csum;
348 PUSHORT csum_ptr;
349 USHORT remaining;
350 USHORT ip4_length;
351 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
353 //FUNCTION_ENTER();
355 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
356 ASSERT(mdl);
358 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
359 data_length = ip4_length + XN_HDR_SIZE;
361 if ((USHORT)data_length > total_length)
362 {
363 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
364 return FALSE;
365 }
367 switch (pi->ip_proto)
368 {
369 case 6:
370 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
371 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
372 break;
373 case 17:
374 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
375 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
376 break;
377 default:
378 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
379 //FUNCTION_EXIT();
380 return FALSE; // should never happen
381 }
383 if (set_csum)
384 *csum_ptr = 0;
386 csum = 0;
387 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
388 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
389 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
391 remaining = ip4_length - pi->ip4_header_length;
393 csum += remaining;
395 csum_span = FALSE;
396 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
397 while (i < data_length)
398 {
399 /* don't include the checksum field itself in the calculation */
400 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
401 {
402 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
403 i += 2;
404 buffer_offset += 2;
405 continue;
406 }
407 if (csum_span)
408 {
409 /* the other half of the next bit */
410 ASSERT(buffer_offset == 0);
411 csum += (USHORT)buffer[buffer_offset];
412 csum_span = FALSE;
413 i += 1;
414 buffer_offset += 1;
415 }
416 else if (buffer_offset == buffer_length - 1)
417 {
418 /* deal with a buffer ending on an odd byte boundary */
419 csum += (USHORT)buffer[buffer_offset] << 8;
420 csum_span = TRUE;
421 i += 1;
422 buffer_offset += 1;
423 }
424 else
425 {
426 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
427 i += 2;
428 buffer_offset += 2;
429 }
430 if (buffer_offset == buffer_length && i < total_length)
431 {
432 NdisGetNextBuffer(mdl, &mdl);
433 if (mdl == NULL)
434 {
435 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
436 return FALSE; // should never happen
437 }
438 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
439 ASSERT(buffer_length);
440 buffer_offset = 0;
441 }
442 }
444 while (csum & 0xFFFF0000)
445 csum = (csum & 0xFFFF) + (csum >> 16);
447 if (set_csum)
448 {
449 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
450 }
451 else
452 {
453 //FUNCTION_EXIT();
454 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
455 }
456 //FUNCTION_EXIT();
457 return TRUE;
458 }
460 static ULONG
461 XenNet_MakePackets(
462 struct xennet_info *xi,
463 PLIST_ENTRY rx_packet_list,
464 packet_info_t *pi
465 )
466 {
467 ULONG packet_count = 0;
468 PNDIS_PACKET packet;
469 PLIST_ENTRY entry;
470 UCHAR psh;
471 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
472 ULONG parse_result;
473 //PNDIS_BUFFER buffer;
474 shared_buffer_t *page_buf;
476 //FUNCTION_ENTER();
478 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
480 if (!XenNet_FilterAcceptPacket(xi, pi))
481 {
482 goto done;
483 }
485 switch (pi->ip_proto)
486 {
487 case 6: // TCP
488 if (pi->split_required)
489 break;
490 // fallthrough
491 case 17: // UDP
492 packet = XenNet_MakePacket(xi, pi);
493 if (packet == NULL)
494 {
495 //KdPrint((__DRIVER_NAME " Ran out of packets\n"));
496 xi->stat_rx_no_buffer++;
497 packet_count = 0;
498 goto done;
499 }
500 if (parse_result == PARSE_OK)
501 {
502 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
503 packet, TcpIpChecksumPacketInfo);
504 if (pi->csum_blank || pi->data_validated)
505 {
506 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
507 {
508 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
509 {
510 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
511 }
512 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
513 {
514 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
515 }
516 if (pi->csum_blank)
517 {
518 XenNet_SumPacketData(pi, packet, TRUE);
519 }
520 }
521 else if (xi->config_csum_rx_check)
522 {
523 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
524 {
525 if (XenNet_SumPacketData(pi, packet, FALSE))
526 {
527 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
528 }
529 else
530 {
531 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
532 }
533 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
534 {
535 if (XenNet_SumPacketData(pi, packet, FALSE))
536 {
537 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
538 }
539 else
540 {
541 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
542 }
543 }
544 }
545 }
546 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
547 InsertTailList(rx_packet_list, entry);
548 packet_count = 1;
549 goto done;
550 default:
551 packet = XenNet_MakePacket(xi, pi);
552 if (packet == NULL)
553 {
554 //KdPrint((__DRIVER_NAME " Ran out of packets\n"));
555 xi->stat_rx_no_buffer++;
556 packet_count = 0;
557 goto done;
558 }
559 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
560 InsertTailList(rx_packet_list, entry);
561 packet_count = 1;
562 goto done;
563 }
565 pi->tcp_remaining = pi->tcp_length;
567 /* we can make certain assumptions here as the following code is only for tcp4 */
568 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
569 while (pi->tcp_remaining)
570 {
571 PUCHAR header_va;
572 PMDL mdl;
573 UINT total_length;
574 UINT buffer_length;
575 packet = XenNet_MakePacket(xi, pi);
576 if (!packet)
577 {
578 //KdPrint((__DRIVER_NAME " Ran out of packets\n"));
579 xi->stat_rx_no_buffer++;
580 break; /* we are out of memory - just drop the packets */
581 }
582 if (xi->setting_csum.V4Receive.TcpChecksum)
583 {
584 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
585 packet, TcpIpChecksumPacketInfo);
586 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
587 }
588 if (psh)
589 {
590 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
591 if (pi->tcp_remaining)
592 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
593 else
594 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
595 }
596 XenNet_SumPacketData(pi, packet, TRUE);
597 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
598 InsertTailList(rx_packet_list, entry);
599 packet_count++;
600 }
602 done:
603 page_buf = pi->first_pb;
604 while (page_buf)
605 {
606 shared_buffer_t *next_pb;
608 next_pb = page_buf->next;
609 put_pb_on_freelist(xi, page_buf);
610 page_buf = next_pb;
611 }
612 XenNet_ClearPacketInfo(pi);
613 //FUNCTION_EXIT();
614 return packet_count;
615 }
617 static BOOLEAN
618 XenNet_RxQueueDpcSynchronized(PVOID context)
619 {
620 struct xennet_info *xi = context;
622 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
624 return TRUE;
625 }
627 #define MAXIMUM_PACKETS_PER_INDICATE 32
629 /* We limit the number of packets per interrupt so that acks get a chance
630 under high rx load. The DPC is immediately re-scheduled */
631 #define MAXIMUM_PACKETS_PER_INTERRUPT 32 /* this is calculated before large packet split */
632 #define MAXIMUM_DATA_PER_INTERRUPT (MAXIMUM_PACKETS_PER_INTERRUPT * 1500) /* help account for large packets */
634 // Called at DISPATCH_LEVEL
635 static VOID
636 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
637 {
638 struct xennet_info *xi = context;
639 RING_IDX cons, prod;
640 LIST_ENTRY rx_packet_list;
641 PLIST_ENTRY entry;
642 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
643 ULONG packet_count = 0;
644 ULONG buffer_count = 0;
645 ULONG packet_data = 0;
646 ULONG interim_packet_data = 0;
647 struct netif_extra_info *ei;
648 USHORT id;
649 int more_to_do = FALSE;
650 packet_info_t *pi = &xi->rxpi[KeGetCurrentProcessorNumber() & 0xff];
651 //NDIS_STATUS status;
652 shared_buffer_t *page_buf;
653 shared_buffer_t *head_buf = NULL;
654 shared_buffer_t *tail_buf = NULL;
655 shared_buffer_t *last_buf = NULL;
656 BOOLEAN extra_info_flag = FALSE;
657 BOOLEAN more_data_flag = FALSE;
658 PNDIS_BUFFER buffer;
660 UNREFERENCED_PARAMETER(dpc);
661 UNREFERENCED_PARAMETER(arg1);
662 UNREFERENCED_PARAMETER(arg2);
664 //FUNCTION_ENTER();
666 if (!xi->connected)
667 return; /* a delayed DPC could let this come through... just do nothing */
669 InitializeListHead(&rx_packet_list);
671 /* get all the buffers off the ring as quickly as possible so the lock is held for a minimum amount of time */
673 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
675 if (xi->rx_shutting_down)
676 {
677 /* there is a chance that our Dpc had been queued just before the shutdown... */
678 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
679 return;
680 }
682 if (xi->rx_partial_buf)
683 {
684 head_buf = xi->rx_partial_buf;
685 tail_buf = xi->rx_partial_buf;
686 while (tail_buf->next)
687 tail_buf = tail_buf->next;
688 more_data_flag = xi->rx_partial_more_data_flag;
689 extra_info_flag = xi->rx_partial_extra_info_flag;
690 xi->rx_partial_buf = NULL;
691 }
693 do {
694 prod = xi->rx.sring->rsp_prod;
695 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
697 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAXIMUM_PACKETS_PER_INTERRUPT && packet_data < MAXIMUM_DATA_PER_INTERRUPT; cons++)
698 {
699 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
700 page_buf = xi->rx_ring_pbs[id];
701 ASSERT(page_buf);
702 xi->rx_ring_pbs[id] = NULL;
703 xi->rx_id_free++;
704 memcpy(&page_buf->rsp, RING_GET_RESPONSE(&xi->rx, cons), max(sizeof(struct netif_rx_response), sizeof(struct netif_extra_info)));
705 if (!extra_info_flag)
706 {
707 if (page_buf->rsp.status <= 0
708 || page_buf->rsp.offset + page_buf->rsp.status > PAGE_SIZE)
709 {
710 KdPrint((__DRIVER_NAME " Error: rsp offset %d, size %d\n",
711 page_buf->rsp.offset, page_buf->rsp.status));
712 ASSERT(!extra_info_flag);
713 put_pb_on_freelist(xi, page_buf);
714 continue;
715 }
716 }
718 if (!head_buf)
719 {
720 head_buf = page_buf;
721 tail_buf = page_buf;
722 }
723 else
724 {
725 tail_buf->next = page_buf;
726 tail_buf = page_buf;
727 }
728 page_buf->next = NULL;
730 if (extra_info_flag)
731 {
732 ei = (struct netif_extra_info *)&page_buf->rsp;
733 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
734 }
735 else
736 {
737 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
738 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
739 interim_packet_data += page_buf->rsp.status;
740 }
742 if (!extra_info_flag && !more_data_flag)
743 {
744 last_buf = page_buf;
745 packet_count++;
746 packet_data += interim_packet_data;
747 interim_packet_data = 0;
748 }
749 buffer_count++;
750 }
751 xi->rx.rsp_cons = cons;
753 /* Give netback more buffers */
754 XenNet_FillRing(xi);
756 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
757 break;
759 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
760 if (!more_to_do)
761 {
762 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
763 KeMemoryBarrier();
764 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
765 }
766 } while (more_to_do);
768 /* anything past last_buf belongs to an incomplete packet... */
769 if (last_buf && last_buf->next)
770 {
771 KdPrint((__DRIVER_NAME " Partial receive\n"));
772 xi->rx_partial_buf = last_buf->next;
773 xi->rx_partial_more_data_flag = more_data_flag;
774 xi->rx_partial_extra_info_flag = extra_info_flag;
775 last_buf->next = NULL;
776 }
778 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
780 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
781 {
782 /* fire again immediately */
783 //KdPrint((__DRIVER_NAME " Dpc Duration Exceeded\n"));
784 dpc_limit_hit++;
785 /* we want the Dpc on the end of the queue. By definition we are already on the right CPU so we know the Dpc queue will be run immediately */
786 KeSetImportanceDpc(&xi->rx_dpc, MediumImportance);
787 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
788 //xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
789 }
790 else
791 {
792 /* make sure the Dpc queue is run immediately next interrupt */
793 KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
794 }
796 /* make packets out of the buffers */
797 page_buf = head_buf;
798 extra_info_flag = FALSE;
799 more_data_flag = FALSE;
800 while (page_buf)
801 {
802 shared_buffer_t *next_buf = page_buf->next;
804 page_buf->next = NULL;
805 if (extra_info_flag)
806 {
807 //KdPrint((__DRIVER_NAME " processing extra info\n"));
808 ei = (struct netif_extra_info *)&page_buf->rsp;
809 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
810 switch (ei->type)
811 {
812 case XEN_NETIF_EXTRA_TYPE_GSO:
813 switch (ei->u.gso.type)
814 {
815 case XEN_NETIF_GSO_TYPE_TCPV4:
816 pi->mss = ei->u.gso.size;
817 //KdPrint((__DRIVER_NAME " mss = %d\n", pi->mss));
818 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
819 break;
820 default:
821 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
822 break;
823 }
824 break;
825 default:
826 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
827 break;
828 }
829 put_pb_on_freelist(xi, page_buf);
830 }
831 else
832 {
833 ASSERT(!page_buf->rsp.offset);
834 if (!more_data_flag) // handling the packet's 1st buffer
835 {
836 if (page_buf->rsp.flags & NETRXF_csum_blank)
837 pi->csum_blank = TRUE;
838 if (page_buf->rsp.flags & NETRXF_data_validated)
839 pi->data_validated = TRUE;
840 }
841 buffer = page_buf->buffer;
842 NdisAdjustBufferLength(buffer, page_buf->rsp.status);
843 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
844 if (pi->first_pb)
845 {
846 ASSERT(pi->curr_pb);
847 //KdPrint((__DRIVER_NAME " additional buffer\n"));
848 pi->curr_pb->next = page_buf;
849 pi->curr_pb = page_buf;
850 ASSERT(pi->curr_buffer);
851 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
852 pi->curr_buffer = buffer;
853 }
854 else
855 {
856 pi->first_pb = page_buf;
857 pi->curr_pb = page_buf;
858 pi->first_buffer = buffer;
859 pi->curr_buffer = buffer;
860 }
861 pi->mdl_count++;
862 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
863 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
864 pi->total_length = pi->total_length + page_buf->rsp.status;
865 }
867 /* Packet done, add it to the list */
868 if (!more_data_flag && !extra_info_flag)
869 {
870 pi->curr_pb = pi->first_pb;
871 pi->curr_buffer = pi->first_buffer;
872 XenNet_MakePackets(xi, &rx_packet_list, pi);
873 }
875 page_buf = next_buf;
876 }
877 ASSERT(!more_data_flag && !extra_info_flag);
879 xi->stat_rx_ok += packet_count;
881 /* indicate packets to NDIS */
882 entry = RemoveHeadList(&rx_packet_list);
883 packet_count = 0;
884 while (entry != &rx_packet_list)
885 {
886 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
887 ASSERT(*(shared_buffer_t **)&packet->MiniportReservedEx[0]);
889 packets[packet_count++] = packet;
890 InterlockedIncrement(&xi->rx_outstanding);
891 entry = RemoveHeadList(&rx_packet_list);
892 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
893 {
894 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
895 packet_count = 0;
896 }
897 }
898 //FUNCTION_EXIT();
899 }
901 /* called at DISPATCH_LEVEL */
902 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
903 VOID
904 XenNet_ReturnPacket(
905 IN NDIS_HANDLE MiniportAdapterContext,
906 IN PNDIS_PACKET Packet
907 )
908 {
909 struct xennet_info *xi = MiniportAdapterContext;
910 PNDIS_BUFFER buffer;
911 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[0];
913 //FUNCTION_ENTER();
915 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
917 NdisUnchainBufferAtFront(Packet, &buffer);
919 while (buffer)
920 {
921 shared_buffer_t *next_buf;
922 ASSERT(page_buf);
923 next_buf = page_buf->next;
924 if (!page_buf->virtual)
925 {
926 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
927 PUCHAR va;
928 UINT len;
929 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
930 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
931 NdisFreeBuffer(buffer);
932 }
933 else
934 {
935 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
936 if (buffer != page_buf->buffer)
937 NdisFreeBuffer(buffer);
938 put_pb_on_freelist(xi, page_buf);
939 }
940 NdisUnchainBufferAtFront(Packet, &buffer);
941 page_buf = next_buf;
942 }
944 put_packet_on_freelist(xi, Packet);
945 InterlockedDecrement(&xi->rx_outstanding);
947 if (!xi->rx_outstanding && xi->rx_shutting_down)
948 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
950 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
952 XenNet_FillRing(xi);
954 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
956 //FUNCTION_EXIT();
957 }
959 /*
960 Free all Rx buffers (on halt, for example)
961 The ring must be stopped at this point.
962 */
964 static VOID
965 XenNet_PurgeRing(struct xennet_info *xi)
966 {
967 int i;
968 for (i = 0; i < NET_RX_RING_SIZE; i++)
969 {
970 if (xi->rx_ring_pbs[i] != NULL)
971 {
972 put_pb_on_freelist(xi, xi->rx_ring_pbs[i]);
973 xi->rx_ring_pbs[i] = NULL;
974 }
975 }
976 }
978 static VOID
979 XenNet_BufferFree(struct xennet_info *xi)
980 {
981 shared_buffer_t *pb;
983 XenNet_PurgeRing(xi);
985 while ((pb = get_pb_from_freelist(xi)) != NULL)
986 {
987 NdisFreeBuffer(pb->buffer);
988 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
989 pb->gref, FALSE, (ULONG)'XNRX');
990 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
991 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
992 }
993 }
995 VOID
996 XenNet_RxResumeStart(xennet_info_t *xi)
997 {
998 KIRQL old_irql;
1000 FUNCTION_ENTER();
1002 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1003 XenNet_PurgeRing(xi);
1004 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1006 FUNCTION_EXIT();
1009 VOID
1010 XenNet_BufferAlloc(xennet_info_t *xi)
1012 //NDIS_STATUS status;
1013 int i;
1015 xi->rx_id_free = NET_RX_RING_SIZE;
1016 xi->rx_outstanding = 0;
1018 for (i = 0; i < NET_RX_RING_SIZE; i++)
1020 xi->rx_ring_pbs[i] = NULL;
1024 VOID
1025 XenNet_RxResumeEnd(xennet_info_t *xi)
1027 KIRQL old_irql;
1029 FUNCTION_ENTER();
1031 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1032 //XenNet_BufferAlloc(xi);
1033 XenNet_FillRing(xi);
1034 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1036 FUNCTION_EXIT();
1039 BOOLEAN
1040 XenNet_RxInit(xennet_info_t *xi)
1042 NDIS_STATUS status;
1044 FUNCTION_ENTER();
1046 xi->rx_shutting_down = FALSE;
1047 KeInitializeSpinLock(&xi->rx_lock);
1048 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
1049 KeInitializeTimer(&xi->rx_timer);
1050 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
1051 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
1052 KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
1053 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
1054 status = NdisAllocateMemoryWithTag((PVOID)&xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount(), XENNET_POOL_TAG);
1055 if (status != NDIS_STATUS_SUCCESS)
1057 KdPrint(("NdisAllocateMemoryWithTag failed with 0x%x\n", status));
1058 return FALSE;
1060 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount());
1062 stack_new(&xi->rx_pb_stack, NET_RX_RING_SIZE * 4);
1064 XenNet_BufferAlloc(xi);
1066 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
1067 PROTOCOL_RESERVED_SIZE_IN_PACKET);
1068 if (status != NDIS_STATUS_SUCCESS)
1070 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
1071 return FALSE;
1074 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0,
1075 MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH + sizeof(shared_buffer_t), XENNET_POOL_TAG, 0);
1077 XenNet_FillRing(xi);
1079 FUNCTION_EXIT();
1081 return TRUE;
1084 BOOLEAN
1085 XenNet_RxShutdown(xennet_info_t *xi)
1087 KIRQL old_irql;
1089 FUNCTION_ENTER();
1091 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1092 xi->rx_shutting_down = TRUE;
1093 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1095 if (xi->config_rx_interrupt_moderation)
1097 KeCancelTimer(&xi->rx_timer);
1100 KeRemoveQueueDpc(&xi->rx_dpc);
1101 #if (NTDDI_VERSION >= NTDDI_WINXP)
1102 KeFlushQueuedDpcs();
1103 #endif
1105 while (xi->rx_outstanding)
1107 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1108 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1111 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1113 NdisFreeMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount(), 0);
1115 XenNet_BufferFree(xi);
1117 NdisFreePacketPool(xi->rx_packet_pool);
1119 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1121 stack_delete(xi->rx_pb_stack, NULL, NULL);
1122 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1124 FUNCTION_EXIT();
1126 return TRUE;