win-pvdrivers

view xennet/xennet5_rx.c @ 979:8f483a2b2991

Fix up PREfast warnings
author James Harper <james.harper@bendigoit.com.au>
date Sun Apr 15 19:47:10 2012 +1000 (2012-04-15)
parents 941699790045
children 0879eec3f528
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet5.h"
23 static __inline shared_buffer_t *
24 get_pb_from_freelist(struct xennet_info *xi)
25 {
26 NDIS_STATUS status;
27 shared_buffer_t *pb;
28 PVOID ptr_ref;
30 if (stack_pop(xi->rx_pb_stack, &ptr_ref))
31 {
32 pb = ptr_ref;
33 pb->ref_count = 1;
34 InterlockedDecrement(&xi->rx_pb_free);
35 return pb;
36 }
38 /* don't allocate a new one if we are shutting down */
39 if (xi->shutting_down)
40 return NULL;
42 status = NdisAllocateMemoryWithTag(&pb, sizeof(shared_buffer_t), XENNET_POOL_TAG);
43 if (status != STATUS_SUCCESS)
44 {
45 return NULL;
46 }
47 status = NdisAllocateMemoryWithTag(&pb->virtual, PAGE_SIZE, XENNET_POOL_TAG);
48 if (status != STATUS_SUCCESS)
49 {
50 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
51 return NULL;
52 }
53 pb->gref = (grant_ref_t)xi->vectors.GntTbl_GrantAccess(xi->vectors.context,
54 (ULONG)(MmGetPhysicalAddress(pb->virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
55 if (pb->gref == INVALID_GRANT_REF)
56 {
57 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
58 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
59 return NULL;
60 }
61 pb->offset = (USHORT)(ULONG_PTR)pb->virtual & (PAGE_SIZE - 1);
62 NdisAllocateBuffer(&status, &pb->buffer, xi->rx_buffer_pool, (PUCHAR)pb->virtual, PAGE_SIZE);
63 if (status != STATUS_SUCCESS)
64 {
65 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
66 pb->gref, FALSE, (ULONG)'XNRX');
67 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
68 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
69 return NULL;
70 }
71 pb->ref_count = 1;
72 return pb;
73 }
75 static __inline VOID
76 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
77 {
78 UNREFERENCED_PARAMETER(xi);
79 InterlockedIncrement(&pb->ref_count);
80 }
82 static __inline VOID
83 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
84 {
85 if (InterlockedDecrement(&pb->ref_count) == 0)
86 {
87 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
88 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
89 pb->next = NULL;
90 stack_push(xi->rx_pb_stack, pb);
91 InterlockedIncrement(&xi->rx_pb_free);
92 }
93 }
95 // Called at DISPATCH_LEVEL with rx lock held
96 static NDIS_STATUS
97 XenNet_FillRing(struct xennet_info *xi)
98 {
99 unsigned short id;
100 shared_buffer_t *page_buf;
101 ULONG i, notify;
102 ULONG batch_target;
103 RING_IDX req_prod = xi->rx.req_prod_pvt;
104 netif_rx_request_t *req;
106 //FUNCTION_ENTER();
108 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
110 if (batch_target < (xi->rx_target >> 2))
111 {
112 //FUNCTION_EXIT();
113 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
114 }
116 for (i = 0; i < batch_target; i++)
117 {
118 page_buf = get_pb_from_freelist(xi);
119 if (!page_buf)
120 {
121 KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
122 break;
123 }
124 xi->rx_id_free--;
126 /* Give to netback */
127 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
128 ASSERT(xi->rx_ring_pbs[id] == NULL);
129 xi->rx_ring_pbs[id] = page_buf;
130 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
131 req->id = id;
132 req->gref = page_buf->gref;
133 ASSERT(req->gref != INVALID_GRANT_REF);
134 }
135 KeMemoryBarrier();
136 xi->rx.req_prod_pvt = req_prod + i;
137 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
138 if (notify)
139 {
140 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
141 }
143 //FUNCTION_EXIT();
145 return NDIS_STATUS_SUCCESS;
146 }
148 /* lock free */
149 static PNDIS_PACKET
150 get_packet_from_freelist(struct xennet_info *xi)
151 {
152 NDIS_STATUS status;
153 PNDIS_PACKET packet;
154 PVOID ptr_ref;
156 if (stack_pop(xi->rx_packet_stack, &ptr_ref))
157 {
158 packet = ptr_ref;
159 return packet;
160 }
162 if (xi->rx_shutting_down) /* don't keep allocating new packets on shutdown */
163 return NULL;
165 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
166 if (status != NDIS_STATUS_SUCCESS)
167 {
168 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
169 return NULL;
170 }
171 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
172 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
173 return packet;
174 }
176 /* lock free */
177 static VOID
178 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
179 {
180 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
182 UNREFERENCED_PARAMETER(xi);
183 NdisReinitializePacket(packet);
184 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
185 packet, TcpIpChecksumPacketInfo);
186 csum_info->Value = 0;
188 stack_push(xi->rx_packet_stack, packet);
189 }
191 static PNDIS_PACKET
192 XenNet_MakePacket(struct xennet_info *xi, packet_info_t *pi)
193 {
194 NDIS_STATUS status;
195 PNDIS_PACKET packet;
196 PNDIS_BUFFER out_buffer;
197 USHORT new_ip4_length;
198 PUCHAR header_va;
199 ULONG out_remaining;
200 ULONG tcp_length;
201 ULONG header_extra;
202 shared_buffer_t *header_buf;
204 //FUNCTION_ENTER();
206 packet = get_packet_from_freelist(xi);
207 if (packet == NULL)
208 {
209 /* buffers will be freed in MakePackets */
210 //KdPrint((__DRIVER_NAME " No free packets\n"));
211 //FUNCTION_EXIT();
212 return NULL;
213 }
215 if (!pi->split_required && pi->mdl_count == 1)
216 {
217 /* shortcut for the single packet single mdl case */
219 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
220 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->first_buffer, 0, pi->total_length);
221 if (status != STATUS_SUCCESS)
222 {
223 KdPrint((__DRIVER_NAME " No free rx buffers\n"));
224 put_packet_on_freelist(xi, packet);
225 return NULL;
226 }
227 NdisChainBufferAtBack(packet, out_buffer);
228 *(shared_buffer_t **)&packet->MiniportReservedEx[0] = pi->first_pb;
229 ref_pb(xi, pi->first_pb); /* so that the buffer doesn't get freed at the end of MakePackets*/
230 //FUNCTION_EXIT();
231 /* windows gets lazy about ack packets and holds on to them forever under high load situations. we don't like this */
232 if (pi->ip_proto == 6 && pi->tcp_length == 0)
233 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
234 else
235 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
236 return packet;
237 }
239 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
240 if (!header_buf)
241 {
242 KdPrint((__DRIVER_NAME " No free header buffers\n"));
243 put_packet_on_freelist(xi, packet);
244 return NULL;
245 }
246 header_va = (PUCHAR)(header_buf + 1);
247 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
248 NdisMoveMemory(header_va, pi->header, pi->header_length);
250 /* make sure we satisfy the lookahead requirement */
252 if (pi->split_required)
253 {
254 /* for split packets we need to make sure the 'header' is no bigger than header+mss bytes */
255 XenNet_BuildHeader(pi, header_va, min((ULONG)MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length + pi->mss, MAX_ETH_HEADER_LENGTH + max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead)));
256 }
257 else
258 {
259 XenNet_BuildHeader(pi, header_va, max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead) + MAX_ETH_HEADER_LENGTH);
260 }
261 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
262 ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
263 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
264 if (status != STATUS_SUCCESS)
265 {
266 KdPrint((__DRIVER_NAME " No free header buffers\n"));
267 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, header_buf);
268 put_packet_on_freelist(xi, packet);
269 return NULL;
270 }
271 NdisChainBufferAtBack(packet, out_buffer);
272 *(shared_buffer_t **)&packet->MiniportReservedEx[0] = header_buf;
273 header_buf->next = pi->curr_pb;
275 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too... maybe
277 if (pi->split_required)
278 {
279 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
280 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
281 //KdPrint((__DRIVER_NAME " new_ip4_length = %d\n", new_ip4_length));
282 //KdPrint((__DRIVER_NAME " this tcp_length = %d\n", tcp_length));
283 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
284 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
285 pi->tcp_seq += tcp_length;
286 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
287 /* part of the packet is already present in the header buffer for lookahead */
288 out_remaining = tcp_length - header_extra;
289 ASSERT((LONG)out_remaining >= 0);
290 }
291 else
292 {
293 out_remaining = pi->total_length - pi->header_length;
294 ASSERT((LONG)out_remaining >= 0);
295 }
296 //KdPrint((__DRIVER_NAME " before loop - out_remaining = %d\n", out_remaining));
298 while (out_remaining != 0)
299 {
300 ULONG in_buffer_offset;
301 ULONG in_buffer_length;
302 ULONG out_length;
304 //KdPrint((__DRIVER_NAME " in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_buffer, pi->curr_pb));
305 if (!pi->curr_buffer || !pi->curr_pb)
306 {
307 KdPrint((__DRIVER_NAME " out of buffers for packet\n"));
308 KdPrint((__DRIVER_NAME " out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_buffer, pi->curr_pb));
309 // TODO: free some stuff or we'll leak
310 /* unchain buffers then free packet */
311 return NULL;
312 }
313 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
314 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
315 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
316 ASSERT(status == STATUS_SUCCESS); //TODO: properly handle error
317 NdisChainBufferAtBack(packet, out_buffer);
318 ref_pb(xi, pi->curr_pb);
319 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
320 if (pi->curr_mdl_offset == in_buffer_length)
321 {
322 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
323 pi->curr_pb = pi->curr_pb->next;
324 pi->curr_mdl_offset = 0;
325 }
326 out_remaining -= out_length;
327 }
328 if (pi->split_required)
329 {
330 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
331 }
332 if (header_extra > 0)
333 pi->header_length -= header_extra;
334 ASSERT(*(shared_buffer_t **)&packet->MiniportReservedEx[0]);
335 /* windows gets lazy about ack packets and holds on to them forever under high load situations. we don't like this */
336 if (pi->ip_proto == 6 && pi->tcp_length == 0)
337 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_RESOURCES);
338 else
339 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
340 //FUNCTION_EXIT();
341 return packet;
342 }
344 /*
345 Windows appears to insist that the checksum on received packets is correct, and won't
346 believe us when we lie about it, which happens when the packet is generated on the
347 same bridge in Dom0. Doh!
348 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
349 */
351 static BOOLEAN
352 XenNet_SumPacketData(
353 packet_info_t *pi,
354 PNDIS_PACKET packet,
355 BOOLEAN set_csum
356 )
357 {
358 USHORT i;
359 PUCHAR buffer;
360 PMDL mdl;
361 UINT total_length;
362 UINT data_length;
363 UINT buffer_length;
364 USHORT buffer_offset;
365 ULONG csum;
366 PUSHORT csum_ptr;
367 USHORT remaining;
368 USHORT ip4_length;
369 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
371 //FUNCTION_ENTER();
373 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
374 if (!buffer) {
375 FUNCTION_MSG("NdisGetFirstBufferFromPacketSafe failed, buffer == NULL\n");
376 return FALSE;
377 }
378 ASSERT(mdl);
380 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
381 data_length = ip4_length + XN_HDR_SIZE;
383 if ((USHORT)data_length > total_length) {
384 FUNCTION_MSG("Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length);
385 return FALSE;
386 }
388 switch (pi->ip_proto)
389 {
390 case 6:
391 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
392 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
393 break;
394 case 17:
395 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
396 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
397 break;
398 default:
399 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
400 //FUNCTION_EXIT();
401 return FALSE; // should never happen
402 }
404 if (set_csum)
405 *csum_ptr = 0;
407 csum = 0;
408 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
409 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
410 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
412 remaining = ip4_length - pi->ip4_header_length;
414 csum += remaining;
416 csum_span = FALSE;
417 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
418 while (i < data_length)
419 {
420 /* don't include the checksum field itself in the calculation */
421 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
422 {
423 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
424 i += 2;
425 buffer_offset += 2;
426 continue;
427 }
428 if (csum_span)
429 {
430 /* the other half of the next bit */
431 ASSERT(buffer_offset == 0);
432 csum += (USHORT)buffer[buffer_offset];
433 csum_span = FALSE;
434 i += 1;
435 buffer_offset += 1;
436 }
437 else if (buffer_offset == buffer_length - 1)
438 {
439 /* deal with a buffer ending on an odd byte boundary */
440 csum += (USHORT)buffer[buffer_offset] << 8;
441 csum_span = TRUE;
442 i += 1;
443 buffer_offset += 1;
444 }
445 else
446 {
447 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
448 i += 2;
449 buffer_offset += 2;
450 }
451 if (buffer_offset == buffer_length && i < total_length)
452 {
453 NdisGetNextBuffer(mdl, &mdl);
454 if (mdl == NULL)
455 {
456 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
457 return FALSE; // should never happen
458 }
459 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
460 ASSERT(buffer_length);
461 buffer_offset = 0;
462 }
463 }
465 while (csum & 0xFFFF0000)
466 csum = (csum & 0xFFFF) + (csum >> 16);
468 if (set_csum)
469 {
470 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
471 }
472 else
473 {
474 //FUNCTION_EXIT();
475 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
476 }
477 //FUNCTION_EXIT();
478 return TRUE;
479 }
481 static ULONG
482 XenNet_MakePackets(
483 struct xennet_info *xi,
484 PLIST_ENTRY rx_packet_list,
485 packet_info_t *pi
486 )
487 {
488 ULONG packet_count = 0;
489 PNDIS_PACKET packet;
490 PLIST_ENTRY entry;
491 UCHAR psh;
492 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
493 ULONG parse_result;
494 //PNDIS_BUFFER buffer;
495 shared_buffer_t *page_buf;
497 //FUNCTION_ENTER();
499 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
501 if (!XenNet_FilterAcceptPacket(xi, pi))
502 {
503 goto done;
504 }
506 switch (pi->ip_proto)
507 {
508 case 6: // TCP
509 if (pi->split_required)
510 break;
511 // fallthrough
512 case 17: // UDP
513 packet = XenNet_MakePacket(xi, pi);
514 if (packet == NULL)
515 {
516 //KdPrint((__DRIVER_NAME " Ran out of packets\n"));
517 xi->stat_rx_no_buffer++;
518 packet_count = 0;
519 goto done;
520 }
521 if (parse_result == PARSE_OK)
522 {
523 BOOLEAN checksum_offload = FALSE;
524 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
525 packet, TcpIpChecksumPacketInfo);
526 ASSERT(csum_info->Value == 0);
527 if (pi->csum_blank || pi->data_validated)
528 {
529 /* we know this is IPv4, and we know Linux always validates the IPv4 checksum for us */
530 if (xi->setting_csum.V4Receive.IpChecksum)
531 {
532 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported)
533 {
534 if (XenNet_CheckIpHeader(pi->header, pi->ip4_header_length))
535 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
536 else
537 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
538 }
539 }
540 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
541 {
542 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
543 {
544 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
545 checksum_offload = TRUE;
546 }
547 }
548 else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
549 {
550 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
551 checksum_offload = TRUE;
552 }
553 if (pi->csum_blank && (!xi->config_csum_rx_dont_fix || !checksum_offload))
554 {
555 XenNet_SumPacketData(pi, packet, TRUE);
556 }
557 }
558 else if (xi->config_csum_rx_check && pi->ip_version == 4)
559 {
560 if (xi->setting_csum.V4Receive.IpChecksum)
561 {
562 if (!pi->ip_has_options || xi->setting_csum.V4Receive.IpOptionsSupported)
563 {
564 if (XenNet_CheckIpHeader(pi->header, pi->ip4_header_length))
565 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
566 else
567 csum_info->Receive.NdisPacketIpChecksumFailed = TRUE;
568 }
569 }
570 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
571 {
572 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
573 {
574 if (XenNet_SumPacketData(pi, packet, FALSE))
575 {
576 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
577 }
578 else
579 {
580 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
581 }
582 }
583 }
584 else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
585 {
586 if (XenNet_SumPacketData(pi, packet, FALSE))
587 {
588 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
589 }
590 else
591 {
592 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
593 }
594 }
595 }
596 }
597 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
598 InsertTailList(rx_packet_list, entry);
599 packet_count = 1;
600 goto done;
601 default:
602 packet = XenNet_MakePacket(xi, pi);
603 if (packet == NULL)
604 {
605 //KdPrint((__DRIVER_NAME " Ran out of packets\n"));
606 xi->stat_rx_no_buffer++;
607 packet_count = 0;
608 goto done;
609 }
610 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
611 InsertTailList(rx_packet_list, entry);
612 packet_count = 1;
613 goto done;
614 }
616 pi->tcp_remaining = pi->tcp_length;
618 /* we can make certain assumptions here as the following code is only for tcp4 */
619 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
620 while (pi->tcp_remaining)
621 {
622 PUCHAR header_va;
623 PMDL mdl;
624 UINT total_length;
625 UINT buffer_length;
626 packet = XenNet_MakePacket(xi, pi);
627 if (!packet)
628 {
629 //KdPrint((__DRIVER_NAME " Ran out of packets\n"));
630 xi->stat_rx_no_buffer++;
631 break; /* we are out of memory - just drop the packets */
632 }
633 if (xi->setting_csum.V4Receive.TcpChecksum)
634 {
635 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
636 packet, TcpIpChecksumPacketInfo);
637 csum_info->Receive.NdisPacketIpChecksumSucceeded = TRUE;
638 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
639 }
640 if (psh)
641 {
642 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
643 if (pi->tcp_remaining)
644 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
645 else
646 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
647 }
648 XenNet_SumPacketData(pi, packet, TRUE);
649 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
650 InsertTailList(rx_packet_list, entry);
651 packet_count++;
652 }
654 done:
655 page_buf = pi->first_pb;
656 while (page_buf)
657 {
658 shared_buffer_t *next_pb;
660 next_pb = page_buf->next;
661 put_pb_on_freelist(xi, page_buf);
662 page_buf = next_pb;
663 }
664 XenNet_ClearPacketInfo(pi);
665 //FUNCTION_EXIT();
666 return packet_count;
667 }
670 /* called at DISPATCH_LEVEL */
671 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
672 VOID
673 XenNet_ReturnPacket(
674 IN NDIS_HANDLE MiniportAdapterContext,
675 IN PNDIS_PACKET Packet
676 )
677 {
678 struct xennet_info *xi = MiniportAdapterContext;
679 PNDIS_BUFFER buffer;
680 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[0];
682 //FUNCTION_ENTER();
684 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
686 NdisUnchainBufferAtFront(Packet, &buffer);
688 while (buffer)
689 {
690 shared_buffer_t *next_buf;
691 ASSERT(page_buf);
692 next_buf = page_buf->next;
693 if (!page_buf->virtual)
694 {
695 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
696 PUCHAR va;
697 UINT len;
698 #pragma warning(suppress:28193) /* va is valid because it was mapped earlier */
699 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
700 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
701 NdisFreeBuffer(buffer);
702 }
703 else
704 {
705 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
706 if (buffer != page_buf->buffer)
707 NdisFreeBuffer(buffer);
708 put_pb_on_freelist(xi, page_buf);
709 }
710 NdisUnchainBufferAtFront(Packet, &buffer);
711 page_buf = next_buf;
712 }
714 put_packet_on_freelist(xi, Packet);
715 InterlockedDecrement(&xi->rx_outstanding);
717 if (!xi->rx_outstanding && xi->rx_shutting_down)
718 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
720 #if 0 /* don't do this as it's called an awful lot */
721 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
723 XenNet_FillRing(xi);
725 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
726 #endif
727 //FUNCTION_EXIT();
728 }
730 #define MAXIMUM_PACKETS_PER_INDICATE 32
732 /* We limit the number of packets per interrupt so that acks get a chance
733 under high rx load. The DPC is immediately re-scheduled */
734 #define MAXIMUM_PACKETS_PER_INTERRUPT 32 /* this is calculated before large packet split */
735 #define MAXIMUM_DATA_PER_INTERRUPT (MAXIMUM_PACKETS_PER_INTERRUPT * 1500) /* help account for large packets */
737 // Called at DISPATCH_LEVEL
738 BOOLEAN
739 XenNet_RxBufferCheck(struct xennet_info *xi)
740 {
741 RING_IDX cons, prod;
742 LIST_ENTRY rx_packet_list;
743 LIST_ENTRY rx_header_only_packet_list;
744 PLIST_ENTRY entry;
745 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
746 ULONG packet_count = 0;
747 ULONG buffer_count = 0;
748 ULONG packet_data = 0;
749 ULONG interim_packet_data = 0;
750 struct netif_extra_info *ei;
751 USHORT id;
752 int more_to_do = FALSE;
753 packet_info_t *pi = &xi->rxpi[KeGetCurrentProcessorNumber() & 0xff];
754 shared_buffer_t *page_buf;
755 shared_buffer_t *head_buf = NULL;
756 shared_buffer_t *tail_buf = NULL;
757 shared_buffer_t *last_buf = NULL;
758 BOOLEAN extra_info_flag = FALSE;
759 BOOLEAN more_data_flag = FALSE;
760 PNDIS_BUFFER buffer;
761 BOOLEAN dont_set_event;
763 //FUNCTION_ENTER();
765 if (!xi->connected)
766 return FALSE; /* a delayed DPC could let this come through... just do nothing */
768 InitializeListHead(&rx_packet_list);
770 /* get all the buffers off the ring as quickly as possible so the lock is held for a minimum amount of time */
772 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
774 if (xi->rx_shutting_down)
775 {
776 /* there is a chance that our Dpc had been queued just before the shutdown... */
777 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
778 return FALSE;
779 }
781 if (xi->rx_partial_buf)
782 {
783 head_buf = xi->rx_partial_buf;
784 tail_buf = xi->rx_partial_buf;
785 while (tail_buf->next)
786 tail_buf = tail_buf->next;
787 more_data_flag = xi->rx_partial_more_data_flag;
788 extra_info_flag = xi->rx_partial_extra_info_flag;
789 xi->rx_partial_buf = NULL;
790 }
792 do {
793 prod = xi->rx.sring->rsp_prod;
794 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
796 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAXIMUM_PACKETS_PER_INTERRUPT && packet_data < MAXIMUM_DATA_PER_INTERRUPT; cons++)
797 {
798 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
799 page_buf = xi->rx_ring_pbs[id];
800 ASSERT(page_buf);
801 xi->rx_ring_pbs[id] = NULL;
802 xi->rx_id_free++;
803 memcpy(&page_buf->rsp, RING_GET_RESPONSE(&xi->rx, cons), max(sizeof(struct netif_rx_response), sizeof(struct netif_extra_info)));
804 if (!extra_info_flag)
805 {
806 if (page_buf->rsp.status <= 0
807 || page_buf->rsp.offset + page_buf->rsp.status > PAGE_SIZE)
808 {
809 KdPrint((__DRIVER_NAME " Error: rsp offset %d, size %d\n",
810 page_buf->rsp.offset, page_buf->rsp.status));
811 ASSERT(!extra_info_flag);
812 put_pb_on_freelist(xi, page_buf);
813 continue;
814 }
815 }
817 if (!head_buf)
818 {
819 head_buf = page_buf;
820 tail_buf = page_buf;
821 }
822 else
823 {
824 tail_buf->next = page_buf;
825 tail_buf = page_buf;
826 }
827 page_buf->next = NULL;
829 if (extra_info_flag)
830 {
831 ei = (struct netif_extra_info *)&page_buf->rsp;
832 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
833 }
834 else
835 {
836 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
837 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
838 interim_packet_data += page_buf->rsp.status;
839 }
841 if (!extra_info_flag && !more_data_flag)
842 {
843 last_buf = page_buf;
844 packet_count++;
845 packet_data += interim_packet_data;
846 interim_packet_data = 0;
847 }
848 buffer_count++;
849 }
850 xi->rx.rsp_cons = cons;
852 /* Give netback more buffers */
853 XenNet_FillRing(xi);
855 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
856 break;
858 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
859 if (!more_to_do)
860 {
861 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
862 KeMemoryBarrier();
863 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
864 }
865 } while (more_to_do);
867 /* anything past last_buf belongs to an incomplete packet... */
868 if (last_buf && last_buf->next)
869 {
870 KdPrint((__DRIVER_NAME " Partial receive\n"));
871 xi->rx_partial_buf = last_buf->next;
872 xi->rx_partial_more_data_flag = more_data_flag;
873 xi->rx_partial_extra_info_flag = extra_info_flag;
874 last_buf->next = NULL;
875 }
877 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
879 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
880 {
881 /* fire again immediately */
882 /* we want the Dpc on the end of the queue. By definition we are already on the right CPU so we know the Dpc queue will be run immediately */
883 KeSetImportanceDpc(&xi->rxtx_dpc, MediumImportance);
884 KeInsertQueueDpc(&xi->rxtx_dpc, NULL, NULL);
885 /* dont set an event in TX path */
886 dont_set_event = TRUE;
887 }
888 else
889 {
890 /* make sure the Dpc queue is run immediately next interrupt */
891 KeSetImportanceDpc(&xi->rxtx_dpc, HighImportance);
892 /* set an event in TX path */
893 dont_set_event = FALSE;
894 }
896 /* make packets out of the buffers */
897 page_buf = head_buf;
898 extra_info_flag = FALSE;
899 more_data_flag = FALSE;
900 while (page_buf)
901 {
902 shared_buffer_t *next_buf = page_buf->next;
904 page_buf->next = NULL;
905 if (extra_info_flag)
906 {
907 //KdPrint((__DRIVER_NAME " processing extra info\n"));
908 ei = (struct netif_extra_info *)&page_buf->rsp;
909 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
910 switch (ei->type)
911 {
912 case XEN_NETIF_EXTRA_TYPE_GSO:
913 switch (ei->u.gso.type)
914 {
915 case XEN_NETIF_GSO_TYPE_TCPV4:
916 pi->mss = ei->u.gso.size;
917 //KdPrint((__DRIVER_NAME " mss = %d\n", pi->mss));
918 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
919 break;
920 default:
921 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
922 break;
923 }
924 break;
925 default:
926 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
927 break;
928 }
929 put_pb_on_freelist(xi, page_buf);
930 }
931 else
932 {
933 ASSERT(!page_buf->rsp.offset);
934 if (!more_data_flag) // handling the packet's 1st buffer
935 {
936 if (page_buf->rsp.flags & NETRXF_csum_blank)
937 pi->csum_blank = TRUE;
938 if (page_buf->rsp.flags & NETRXF_data_validated)
939 pi->data_validated = TRUE;
940 }
941 buffer = page_buf->buffer;
942 NdisAdjustBufferLength(buffer, page_buf->rsp.status);
943 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
944 if (pi->first_pb)
945 {
946 ASSERT(pi->curr_pb);
947 //KdPrint((__DRIVER_NAME " additional buffer\n"));
948 pi->curr_pb->next = page_buf;
949 pi->curr_pb = page_buf;
950 ASSERT(pi->curr_buffer);
951 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
952 pi->curr_buffer = buffer;
953 }
954 else
955 {
956 pi->first_pb = page_buf;
957 pi->curr_pb = page_buf;
958 pi->first_buffer = buffer;
959 pi->curr_buffer = buffer;
960 }
961 pi->mdl_count++;
962 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
963 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
964 pi->total_length = pi->total_length + page_buf->rsp.status;
965 }
967 /* Packet done, add it to the list */
968 if (!more_data_flag && !extra_info_flag)
969 {
970 pi->curr_pb = pi->first_pb;
971 pi->curr_buffer = pi->first_buffer;
972 XenNet_MakePackets(xi, &rx_packet_list, pi);
973 }
975 page_buf = next_buf;
976 }
977 ASSERT(!more_data_flag && !extra_info_flag);
979 xi->stat_rx_ok += packet_count;
981 /* indicate packets to NDIS */
982 entry = RemoveHeadList(&rx_packet_list);
983 InitializeListHead(&rx_header_only_packet_list);
984 packet_count = 0;
986 while (entry != &rx_packet_list) {
987 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
988 NDIS_STATUS status;
989 ASSERT(*(shared_buffer_t **)&packet->MiniportReservedEx[0]);
990 status = NDIS_GET_PACKET_STATUS(packet);
991 if (status == NDIS_STATUS_RESOURCES)
992 InsertTailList(&rx_header_only_packet_list, entry);
993 packets[packet_count++] = packet;
994 InterlockedIncrement(&xi->rx_outstanding);
995 entry = RemoveHeadList(&rx_packet_list);
996 /* if we indicate a packet with NDIS_STATUS_RESOURCES then any following packet can't be NDIS_STATUS_SUCCESS */
997 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list
998 || (NDIS_GET_PACKET_STATUS(CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)])) == NDIS_STATUS_SUCCESS
999 && status == NDIS_STATUS_RESOURCES))
1001 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
1002 packet_count = 0;
1005 /* now return the packets for which we indicated NDIS_STATUS_RESOURCES */
1006 entry = RemoveHeadList(&rx_header_only_packet_list);
1007 while (entry != &rx_header_only_packet_list) {
1008 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
1009 entry = RemoveHeadList(&rx_header_only_packet_list);
1010 XenNet_ReturnPacket(xi, packet);
1013 return dont_set_event;
1014 //FUNCTION_EXIT();
1017 /*
1018 Free all Rx buffers (on halt, for example)
1019 The ring must be stopped at this point.
1020 */
1022 static VOID
1023 XenNet_PurgeRing(struct xennet_info *xi)
1025 int i;
1026 for (i = 0; i < NET_RX_RING_SIZE; i++)
1028 if (xi->rx_ring_pbs[i] != NULL)
1030 put_pb_on_freelist(xi, xi->rx_ring_pbs[i]);
1031 xi->rx_ring_pbs[i] = NULL;
1036 static VOID
1037 XenNet_BufferFree(struct xennet_info *xi)
1039 shared_buffer_t *pb;
1041 XenNet_PurgeRing(xi);
1043 while ((pb = get_pb_from_freelist(xi)) != NULL)
1045 NdisFreeBuffer(pb->buffer);
1046 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
1047 pb->gref, FALSE, (ULONG)'XNRX');
1048 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
1049 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
1053 VOID
1054 XenNet_RxResumeStart(xennet_info_t *xi)
1056 KIRQL old_irql;
1058 FUNCTION_ENTER();
1060 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1061 XenNet_PurgeRing(xi);
1062 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1064 FUNCTION_EXIT();
1067 VOID
1068 XenNet_BufferAlloc(xennet_info_t *xi)
1070 //NDIS_STATUS status;
1071 int i;
1073 xi->rx_id_free = NET_RX_RING_SIZE;
1074 xi->rx_outstanding = 0;
1076 for (i = 0; i < NET_RX_RING_SIZE; i++)
1078 xi->rx_ring_pbs[i] = NULL;
1082 VOID
1083 XenNet_RxResumeEnd(xennet_info_t *xi)
1085 KIRQL old_irql;
1087 FUNCTION_ENTER();
1089 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1090 //XenNet_BufferAlloc(xi);
1091 XenNet_FillRing(xi);
1092 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1094 FUNCTION_EXIT();
1097 BOOLEAN
1098 XenNet_RxInit(xennet_info_t *xi)
1100 NDIS_STATUS status;
1102 FUNCTION_ENTER();
1104 xi->rx_shutting_down = FALSE;
1105 KeInitializeSpinLock(&xi->rx_lock);
1106 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
1107 KeInitializeTimer(&xi->rx_timer);
1108 status = NdisAllocateMemoryWithTag((PVOID)&xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount(), XENNET_POOL_TAG);
1109 if (status != NDIS_STATUS_SUCCESS)
1111 KdPrint(("NdisAllocateMemoryWithTag failed with 0x%x\n", status));
1112 return FALSE;
1114 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount());
1116 stack_new(&xi->rx_pb_stack, NET_RX_RING_SIZE * 4);
1118 XenNet_BufferAlloc(xi);
1120 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
1121 PROTOCOL_RESERVED_SIZE_IN_PACKET);
1122 if (status != NDIS_STATUS_SUCCESS)
1124 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
1125 return FALSE;
1127 stack_new(&xi->rx_packet_stack, NET_RX_RING_SIZE * 4);
1129 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0,
1130 MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH + sizeof(shared_buffer_t), XENNET_POOL_TAG, 0);
1132 XenNet_FillRing(xi);
1134 FUNCTION_EXIT();
1136 return TRUE;
1139 BOOLEAN
1140 XenNet_RxShutdown(xennet_info_t *xi)
1142 KIRQL old_irql;
1143 PNDIS_PACKET packet;
1145 FUNCTION_ENTER();
1147 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1148 xi->rx_shutting_down = TRUE;
1149 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1151 if (xi->config_rx_interrupt_moderation)
1153 KeCancelTimer(&xi->rx_timer);
1156 #if (NTDDI_VERSION >= NTDDI_WINXP)
1157 KeFlushQueuedDpcs();
1158 #endif
1160 while (xi->rx_outstanding)
1162 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1163 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1166 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1168 NdisFreeMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount(), 0);
1170 XenNet_BufferFree(xi);
1172 /* this works because get_packet_from_freelist won't allocate new packets when rx_shutting_down */
1173 while ((packet = get_packet_from_freelist(xi)) != NULL)
1174 NdisFreePacket(packet);
1175 stack_delete(xi->rx_packet_stack, NULL, NULL);
1176 NdisFreePacketPool(xi->rx_packet_pool);
1178 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1180 stack_delete(xi->rx_pb_stack, NULL, NULL);
1181 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1183 FUNCTION_EXIT();
1185 return TRUE;