win-pvdrivers

view xennet/xennet_rx.c @ 763:9866a1812579

Fix a network bug where trailing bytes on the ethernet frame caused bad checksums on the RX path
author James Harper <james.harper@bendigoit.com.au>
date Sun Jan 17 15:08:23 2010 +1100 (2010-01-17)
parents 13eb0f02249a
children 6300617040e0
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 /* Not really necessary but keeps PREfast happy */
24 static KDEFERRED_ROUTINE XenNet_RxBufferCheck;
26 static __inline shared_buffer_t *
27 get_pb_from_freelist(struct xennet_info *xi)
28 {
29 shared_buffer_t *pb;
31 if (xi->rx_pb_free == 0)
32 {
33 //KdPrint((__DRIVER_NAME " Out of pb's\n"));
34 return NULL;
35 }
36 xi->rx_pb_free--;
38 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
39 pb->ref_count++;
40 return pb;
41 }
43 static __inline VOID
44 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
45 {
46 UNREFERENCED_PARAMETER(xi);
47 pb->ref_count++;
48 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
49 }
51 static __inline VOID
52 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
53 {
54 pb->ref_count--;
55 //KdPrint((__DRIVER_NAME " decremented pb %p ref to %d\n", pb, pb->ref_count));
56 if (pb->ref_count == 0)
57 {
58 //KdPrint((__DRIVER_NAME " freeing pb %p\n", pb, pb->ref_count));
59 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
60 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
61 pb->next = NULL;
62 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
63 xi->rx_pb_free++;
64 }
65 }
67 // Called at DISPATCH_LEVEL with rx lock held
68 static NDIS_STATUS
69 XenNet_FillRing(struct xennet_info *xi)
70 {
71 unsigned short id;
72 shared_buffer_t *page_buf;
73 ULONG i, notify;
74 ULONG batch_target;
75 RING_IDX req_prod = xi->rx.req_prod_pvt;
76 netif_rx_request_t *req;
78 //FUNCTION_ENTER();
80 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
82 if (batch_target < (xi->rx_target >> 2))
83 {
84 //FUNCTION_EXIT();
85 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
86 }
88 for (i = 0; i < batch_target; i++)
89 {
90 page_buf = get_pb_from_freelist(xi);
91 if (!page_buf)
92 {
93 //KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
94 break;
95 }
96 xi->rx_id_free--;
98 /* Give to netback */
99 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
100 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
101 xi->rx_ring_pbs[id] = page_buf->id;
102 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
103 req->id = id;
104 req->gref = (grant_ref_t)(page_buf->logical.QuadPart >> PAGE_SHIFT);
105 ASSERT(req->gref != INVALID_GRANT_REF);
106 }
107 KeMemoryBarrier();
108 xi->rx.req_prod_pvt = req_prod + i;
109 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
110 if (notify)
111 {
112 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
113 }
115 //FUNCTION_EXIT();
117 return NDIS_STATUS_SUCCESS;
118 }
120 static PNDIS_PACKET
121 get_packet_from_freelist(struct xennet_info *xi)
122 {
123 NDIS_STATUS status;
124 PNDIS_PACKET packet;
126 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
128 if (!xi->rx_packet_free)
129 {
130 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
131 if (status != NDIS_STATUS_SUCCESS)
132 {
133 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
134 return NULL;
135 }
136 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
137 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
138 }
139 else
140 {
141 xi->rx_packet_free--;
142 packet = xi->rx_packet_list[xi->rx_packet_free];
143 }
144 return packet;
145 }
147 static VOID
148 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
149 {
150 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
151 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
153 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
154 {
155 //KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
156 NdisFreePacket(packet);
157 return;
158 }
159 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
160 packet, TcpIpChecksumPacketInfo);
161 csum_info->Value = 0;
162 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
163 xi->rx_packet_list[xi->rx_packet_free] = packet;
164 xi->rx_packet_free++;
165 }
167 static VOID
168 packet_freelist_dispose(struct xennet_info *xi)
169 {
170 while(xi->rx_packet_free != 0)
171 {
172 xi->rx_packet_free--;
173 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
174 }
175 }
177 static PNDIS_PACKET
178 XenNet_MakePacket(struct xennet_info *xi)
179 {
180 NDIS_STATUS status;
181 PNDIS_PACKET packet;
182 PNDIS_BUFFER out_buffer;
183 USHORT new_ip4_length;
184 PUCHAR header_va;
185 packet_info_t *pi = &xi->rxpi;
187 //FUNCTION_ENTER();
189 if (!pi->split_required)
190 {
191 PNDIS_BUFFER buffer;
192 shared_buffer_t *page_buf;
194 //KdPrint((__DRIVER_NAME " !split_required\n"));
196 packet = get_packet_from_freelist(xi);
197 if (packet == NULL)
198 {
199 /* buffers will be freed in MakePackets */
200 KdPrint((__DRIVER_NAME " No free packets\n"));
201 //FUNCTION_EXIT();
202 return NULL;
203 }
204 xi->rx_outstanding++;
206 // what if we needed to consolidate the header? maybe should put that on instead of all the buffers...
207 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = pi->first_pb;
209 buffer = pi->first_buffer;
210 page_buf = pi->first_pb;
211 //KdPrint((__DRIVER_NAME " packet = %p, first_buffer = %p, first_pb = %p\n", packet, buffer, page_buf));
212 while (buffer)
213 {
214 PNDIS_BUFFER next_buffer;
215 //KdPrint((__DRIVER_NAME " buffer = %p\n", buffer));
217 NdisGetNextBuffer(buffer, &next_buffer);
218 NDIS_BUFFER_LINKAGE(buffer) = NULL;
219 NdisChainBufferAtBack(packet, buffer);
220 //ref_pb(xi, page_buf);
222 buffer = next_buffer;
223 page_buf = page_buf->next;
224 }
226 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
227 }
228 else
229 {
230 ULONG out_remaining;
231 shared_buffer_t *header_buf;
233 //KdPrint((__DRIVER_NAME " split_required\n"));
235 packet = get_packet_from_freelist(xi);
236 if (packet == NULL)
237 {
238 //FUNCTION_EXIT();
239 return NULL;
240 }
241 xi->rx_outstanding++;
243 //status = NdisAllocateMemoryWithTag((PUCHAR *)&header_buf, sizeof(shared_buffer_t) + pi->header_length, XENNET_POOL_TAG);
244 ASSERT(sizeof(shared_buffer_t) + pi->header_length < LOOKASIDE_LIST_ALLOC_SIZE);
245 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
246 ASSERT(header_buf); // lazy
247 header_va = (PUCHAR)(header_buf + 1);
248 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
249 NdisMoveMemory(header_va, pi->header, pi->header_length);
251 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too
253 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
254 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
255 NdisChainBufferAtBack(packet, out_buffer);
256 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = header_buf;
257 header_buf->next = pi->curr_pb;
259 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
260 //KdPrint((__DRIVER_NAME " curr_mdl_offset = %d\n", pi->curr_mdl_offset));
261 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
263 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
264 out_remaining = (USHORT)min(pi->mss, pi->tcp_remaining);
265 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + out_remaining);
266 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
267 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
268 pi->tcp_seq += out_remaining;
269 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - out_remaining);
270 do
271 {
272 ULONG in_buffer_offset;
273 ULONG in_buffer_length;
274 ULONG out_length;
275 //UINT tmp_packet_length;
276 //KdPrint((__DRIVER_NAME " curr_buffer = %p\n", pi->curr_buffer));
277 //KdPrint((__DRIVER_NAME " curr_pb = %p\n", pi->curr_pb));
278 //KdPrint((__DRIVER_NAME " out_remaining = %d\n", out_remaining));
279 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
280 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
281 //KdPrint((__DRIVER_NAME " out_length = %d\n", out_length));
282 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
283 //TODO: check status
284 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
285 NdisChainBufferAtBack(packet, out_buffer);
286 //NdisQueryPacketLength(packet, &tmp_packet_length);
287 //KdPrint((__DRIVER_NAME " current packet length = %d\n", tmp_packet_length));
288 ref_pb(xi, pi->curr_pb);
289 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
290 if (pi->curr_mdl_offset == in_buffer_length)
291 {
292 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
293 pi->curr_pb = pi->curr_pb->next;
294 pi->curr_mdl_offset = 0;
295 }
296 out_remaining -= out_length;
297 } while (out_remaining != 0);
298 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
299 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
300 }
302 //FUNCTION_EXIT();
303 return packet;
304 }
306 /*
307 Windows appears to insist that the checksum on received packets is correct, and won't
308 believe us when we lie about it, which happens when the packet is generated on the
309 same bridge in Dom0. Doh!
310 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
311 */
313 static BOOLEAN
314 XenNet_SumPacketData(
315 packet_info_t *pi,
316 PNDIS_PACKET packet,
317 BOOLEAN set_csum
318 )
319 {
320 USHORT i;
321 PUCHAR buffer;
322 PMDL mdl;
323 UINT total_length;
324 UINT data_length;
325 UINT buffer_length;
326 USHORT buffer_offset;
327 ULONG csum;
328 PUSHORT csum_ptr;
329 USHORT remaining;
330 USHORT ip4_length;
331 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
333 //FUNCTION_ENTER();
335 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
336 ASSERT(mdl);
338 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
339 data_length = ip4_length + XN_HDR_SIZE;
341 if ((USHORT)data_length > total_length)
342 {
343 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
344 return FALSE;
345 }
347 switch (pi->ip_proto)
348 {
349 case 6:
350 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
351 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
352 break;
353 case 17:
354 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
355 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
356 break;
357 default:
358 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
359 //FUNCTION_EXIT();
360 return FALSE; // should never happen
361 }
363 if (set_csum)
364 *csum_ptr = 0;
366 csum = 0;
367 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
368 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
369 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
371 remaining = ip4_length - pi->ip4_header_length;
373 csum += remaining;
375 csum_span = FALSE;
376 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
377 while (i < data_length)
378 {
379 /* don't include the checksum field itself in the calculation */
380 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
381 {
382 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
383 i += 2;
384 buffer_offset += 2;
385 continue;
386 }
387 if (csum_span)
388 {
389 /* the other half of the next bit */
390 ASSERT(buffer_offset == 0);
391 csum += (USHORT)buffer[buffer_offset];
392 csum_span = FALSE;
393 i += 1;
394 buffer_offset += 1;
395 }
396 else if (buffer_offset == buffer_length - 1)
397 {
398 /* deal with a buffer ending on an odd byte boundary */
399 csum += (USHORT)buffer[buffer_offset] << 8;
400 csum_span = TRUE;
401 i += 1;
402 buffer_offset += 1;
403 }
404 else
405 {
406 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
407 i += 2;
408 buffer_offset += 2;
409 }
410 if (buffer_offset == buffer_length && i < total_length)
411 {
412 NdisGetNextBuffer(mdl, &mdl);
413 if (mdl == NULL)
414 {
415 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
416 return FALSE; // should never happen
417 }
418 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
419 ASSERT(buffer_length);
420 buffer_offset = 0;
421 }
422 }
424 while (csum & 0xFFFF0000)
425 csum = (csum & 0xFFFF) + (csum >> 16);
427 if (set_csum)
428 {
429 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
430 }
431 else
432 {
433 //FUNCTION_EXIT();
434 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
435 }
436 //FUNCTION_EXIT();
437 return TRUE;
438 }
440 static ULONG
441 XenNet_MakePackets(
442 struct xennet_info *xi,
443 PLIST_ENTRY rx_packet_list
444 )
445 {
446 USHORT i;
447 ULONG packet_count = 0;
448 PNDIS_PACKET packet;
449 PLIST_ENTRY entry;
450 UCHAR psh;
451 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
452 ULONG parse_result;
453 packet_info_t *pi = &xi->rxpi;
454 PNDIS_BUFFER buffer;
455 shared_buffer_t *page_buf;
457 //FUNCTION_ENTER();
459 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
461 //KdPrint((__DRIVER_NAME " ip4_length = %d, tcp_length = %d\n", pi->ip4_length, pi->tcp_length));
463 if ((xi->packet_filter & NDIS_PACKET_TYPE_MULTICAST)
464 && !(xi->packet_filter & NDIS_PACKET_TYPE_ALL_MULTICAST)
465 && (pi->header[0] & 0x01)
466 && !(pi->header[0] == 0xFF && pi->header[1] == 0xFF && pi->header[2] == 0xFF
467 && pi->header[3] == 0xFF && pi->header[4] == 0xFF && pi->header[5] == 0xFF))
468 {
469 for (i = 0; i < xi->multicast_list_size; i++)
470 {
471 if (memcmp(xi->multicast_list[i], pi->header, 6) == 0)
472 break;
473 }
474 if (i == xi->multicast_list_size)
475 {
476 //KdPrint((__DRIVER_NAME " Packet not for my MAC address\n"));
477 goto done;
478 }
479 }
480 switch (pi->ip_proto)
481 {
482 case 6: // TCP
483 if (pi->split_required)
484 break;
485 // fallthrough
486 case 17: // UDP
487 packet = XenNet_MakePacket(xi);
488 if (packet == NULL)
489 {
490 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
491 xi->stat_rx_no_buffer++;
492 packet_count = 0;
493 goto done;
494 }
496 if (parse_result == PARSE_OK)
497 {
498 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
499 packet, TcpIpChecksumPacketInfo);
500 if (pi->csum_blank || pi->data_validated)
501 {
502 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
503 {
504 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
505 {
506 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
507 }
508 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
509 {
510 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
511 }
512 if (pi->csum_blank)
513 {
514 XenNet_SumPacketData(pi, packet, TRUE);
515 }
516 }
517 else if (xi->config_csum_rx_check)
518 {
519 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
520 {
521 if (XenNet_SumPacketData(pi, packet, FALSE))
522 {
523 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
524 }
525 else
526 {
527 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
528 }
529 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
530 {
531 if (XenNet_SumPacketData(pi, packet, FALSE))
532 {
533 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
534 }
535 else
536 {
537 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
538 }
539 }
540 }
541 }
542 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
543 InsertTailList(rx_packet_list, entry);
544 XenNet_ClearPacketInfo(pi);
545 //FUNCTION_EXIT();
546 return 1;
547 default:
548 packet = XenNet_MakePacket(xi);
549 if (packet == NULL)
550 {
551 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
552 xi->stat_rx_no_buffer++;
553 packet_count = 0;
554 goto done;
555 }
556 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
557 InsertTailList(rx_packet_list, entry);
558 XenNet_ClearPacketInfo(pi);
559 //FUNCTION_EXIT();
560 return 1;
561 }
563 /* synchronise the pb with the buffer */
564 buffer = pi->first_buffer;
565 pi->curr_pb = pi->first_pb;
566 //KdPrint((__DRIVER_NAME " first_buffer = %p, curr_buffer = %p, first_pb = %p, curr_pb = %p\n",
567 // pi->first_buffer, pi->curr_buffer, pi->first_pb, pi->curr_pb));
569 while (pi->curr_pb != NULL && buffer != pi->curr_buffer)
570 {
571 NdisGetNextBuffer(buffer, &buffer);
572 pi->curr_pb = pi->curr_pb->next;
573 //KdPrint((__DRIVER_NAME " buffer = %p, curr_pb = %p\n", buffer, pi->curr_pb));
574 }
576 pi->tcp_remaining = pi->tcp_length;
578 /* we can make certain assumptions here as the following code is only for tcp4 */
579 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
580 while (pi->tcp_remaining)
581 {
582 PUCHAR header_va;
583 PMDL mdl;
584 UINT total_length;
585 UINT buffer_length;
586 packet = XenNet_MakePacket(xi);
587 if (!packet)
588 {
589 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
590 xi->stat_rx_no_buffer++;
591 break; /* we are out of memory - just drop the packets */
592 }
593 if (xi->setting_csum.V4Receive.TcpChecksum)
594 {
595 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
596 packet, TcpIpChecksumPacketInfo);
597 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
598 }
599 if (psh)
600 {
601 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
602 if (pi->tcp_remaining)
603 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
604 else
605 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
606 }
607 XenNet_SumPacketData(pi, packet, TRUE);
608 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
609 InsertTailList(rx_packet_list, entry);
610 packet_count++;
611 }
613 done:
614 //buffer = pi->first_buffer;
615 page_buf = pi->first_pb;
616 while (page_buf)
617 {
618 //PNDIS_BUFFER next_buffer;
619 shared_buffer_t *next_pb;
621 //NdisGetNextBuffer(buffer, &next_buffer);
622 next_pb = page_buf->next;
624 //NdisFreeBuffer(buffer);
625 put_pb_on_freelist(xi, page_buf);
627 //buffer = next_buffer;
628 page_buf = next_pb;
629 }
630 XenNet_ClearPacketInfo(pi);
631 //FUNCTION_EXIT();
632 return packet_count;
633 }
635 static BOOLEAN
636 XenNet_RxQueueDpcSynchronized(PVOID context)
637 {
638 struct xennet_info *xi = context;
640 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
642 return TRUE;
643 }
645 #define MAXIMUM_PACKETS_PER_INDICATE 32
646 /*
647 We limit the number of packets per interrupt so that acks get a chance
648 under high rx load. The DPC is immediately re-scheduled */
650 #define MAX_PACKETS_PER_INTERRUPT 64
652 // Called at DISPATCH_LEVEL
653 static VOID
654 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
655 {
656 struct xennet_info *xi = context;
657 RING_IDX cons, prod;
658 LIST_ENTRY rx_packet_list;
659 PLIST_ENTRY entry;
660 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
661 ULONG packet_count = 0;
662 struct netif_rx_response *rxrsp = NULL;
663 struct netif_extra_info *ei;
664 USHORT id;
665 int more_to_do = FALSE;
666 packet_info_t *pi = &xi->rxpi;
667 //NDIS_STATUS status;
668 shared_buffer_t *page_buf;
669 PNDIS_BUFFER buffer;
671 UNREFERENCED_PARAMETER(dpc);
672 UNREFERENCED_PARAMETER(arg1);
673 UNREFERENCED_PARAMETER(arg2);
675 //FUNCTION_ENTER();
677 ASSERT(xi->connected);
679 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
681 if (xi->rx_shutting_down)
682 {
683 /* there is a chance that our Dpc had been queued just before the shutdown... */
684 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
685 return;
686 }
687 InitializeListHead(&rx_packet_list);
689 do {
690 prod = xi->rx.sring->rsp_prod;
691 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
692 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
694 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
695 {
696 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
697 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
698 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
699 xi->rx_ring_pbs[id] = 0xFFFF;
700 xi->rx_id_free++;
701 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
702 if (pi->extra_info)
703 {
704 //KdPrint((__DRIVER_NAME " processing extra info\n"));
705 put_pb_on_freelist(xi, page_buf);
706 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
707 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
708 switch (ei->type)
709 {
710 case XEN_NETIF_EXTRA_TYPE_GSO:
711 switch (ei->u.gso.type)
712 {
713 case XEN_NETIF_GSO_TYPE_TCPV4:
714 pi->mss = ei->u.gso.size;
715 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
716 break;
717 default:
718 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
719 break;
720 }
721 break;
722 default:
723 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
724 break;
725 }
726 }
727 else
728 {
729 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
730 if (rxrsp->status <= 0
731 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
732 {
733 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
734 rxrsp->offset, rxrsp->status));
735 ASSERT(!pi->extra_info);
736 put_pb_on_freelist(xi, page_buf);
737 continue;
738 }
739 ASSERT(!rxrsp->offset);
740 ASSERT(rxrsp->id == id);
741 if (!pi->more_frags) // handling the packet's 1st buffer
742 {
743 if (rxrsp->flags & NETRXF_csum_blank)
744 pi->csum_blank = TRUE;
745 if (rxrsp->flags & NETRXF_data_validated)
746 pi->data_validated = TRUE;
747 }
748 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
749 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
750 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
751 buffer = page_buf->buffer;
752 NdisAdjustBufferLength(buffer, rxrsp->status);
753 if (pi->first_pb)
754 {
755 //KdPrint((__DRIVER_NAME " additional buffer\n"));
756 pi->curr_pb->next = page_buf;
757 pi->curr_pb = page_buf;
758 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
759 pi->curr_buffer = buffer;
760 }
761 else
762 {
763 pi->first_pb = page_buf;
764 pi->curr_pb = page_buf;
765 pi->first_buffer = buffer;
766 pi->curr_buffer = buffer;
767 }
768 pi->mdl_count++;
769 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
770 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
771 pi->total_length = pi->total_length + rxrsp->status;
772 }
774 /* Packet done, add it to the list */
775 if (!pi->more_frags && !pi->extra_info)
776 {
777 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
778 }
779 }
780 xi->rx.rsp_cons = cons;
782 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
783 break;
785 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
786 if (!more_to_do)
787 {
788 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
789 mb();
790 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
791 }
792 } while (more_to_do);
794 if (pi->more_frags || pi->extra_info)
795 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
797 /* Give netback more buffers */
798 XenNet_FillRing(xi);
800 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
801 {
802 /* fire again immediately */
803 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
804 }
806 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
808 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
810 entry = RemoveHeadList(&rx_packet_list);
811 packet_count = 0;
812 while (entry != &rx_packet_list)
813 {
814 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[0]);
815 packets[packet_count++] = packet;
816 entry = RemoveHeadList(&rx_packet_list);
817 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
818 {
819 //KdPrint((__DRIVER_NAME " Indicating\n"));
820 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
821 packet_count = 0;
822 }
823 }
824 //FUNCTION_EXIT();
825 }
827 /* called at DISPATCH_LEVEL */
828 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
829 VOID DDKAPI
830 XenNet_ReturnPacket(
831 IN NDIS_HANDLE MiniportAdapterContext,
832 IN PNDIS_PACKET Packet
833 )
834 {
835 struct xennet_info *xi = MiniportAdapterContext;
836 PNDIS_BUFFER buffer;
837 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[sizeof(LIST_ENTRY)];
839 //FUNCTION_ENTER();
841 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
843 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
845 NdisUnchainBufferAtFront(Packet, &buffer);
846 while (buffer)
847 {
848 shared_buffer_t *next_buf = page_buf->next;
849 if (!page_buf->virtual)
850 {
851 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
852 PUCHAR va;
853 UINT len;
854 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
855 //KdPrint((__DRIVER_NAME " freeing header buffer %p\n", va - sizeof(shared_buffer_t)));
856 //NdisFreeMemory(va - sizeof(shared_buffer_t), len + sizeof(shared_buffer_t), 0);
857 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
858 NdisFreeBuffer(buffer);
859 }
860 else
861 {
862 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
863 if (buffer != page_buf->buffer)
864 NdisFreeBuffer(buffer);
865 put_pb_on_freelist(xi, page_buf);
866 }
867 NdisUnchainBufferAtFront(Packet, &buffer);
868 page_buf = next_buf;
869 }
871 put_packet_on_freelist(xi, Packet);
872 xi->rx_outstanding--;
874 if (!xi->rx_outstanding && xi->rx_shutting_down)
875 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
877 XenNet_FillRing(xi);
879 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
881 //FUNCTION_EXIT();
882 }
884 /*
885 Free all Rx buffers (on halt, for example)
886 The ring must be stopped at this point.
887 */
889 static VOID
890 XenNet_PurgeRing(struct xennet_info *xi)
891 {
892 int i;
893 for (i = 0; i < NET_RX_RING_SIZE; i++)
894 {
895 if (xi->rx_ring_pbs[i] != 0xFFFF)
896 {
897 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
898 xi->rx_ring_pbs[i] = 0xFFFF;
899 }
900 }
901 }
903 static VOID
904 XenNet_BufferFree(struct xennet_info *xi)
905 {
906 shared_buffer_t *pb;
908 XenNet_PurgeRing(xi);
910 while ((pb = get_pb_from_freelist(xi)) != NULL)
911 {
912 NdisFreeBuffer(pb->buffer);
913 NdisMFreeSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, pb->virtual, pb->logical);
914 }
915 }
917 VOID
918 XenNet_RxResumeStart(xennet_info_t *xi)
919 {
920 KIRQL old_irql;
922 FUNCTION_ENTER();
924 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
925 XenNet_PurgeRing(xi);
926 KeReleaseSpinLock(&xi->rx_lock, old_irql);
928 FUNCTION_EXIT();
929 }
931 VOID
932 XenNet_BufferAlloc(xennet_info_t *xi)
933 {
934 NDIS_STATUS status;
935 int i;
937 xi->rx_id_free = NET_RX_RING_SIZE;
938 xi->rx_outstanding = 0;
940 for (i = 0; i < NET_RX_RING_SIZE; i++)
941 {
942 xi->rx_ring_pbs[i] = 0xFFFF;
943 }
945 for (i = 0; i < RX_PAGE_BUFFERS; i++)
946 {
947 xi->rx_pbs[i].id = (USHORT)i;
948 NdisMAllocateSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, &xi->rx_pbs[i].virtual, &xi->rx_pbs[i].logical);
949 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
950 if (status != STATUS_SUCCESS)
951 break;
952 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
953 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
954 }
955 if (i == 0)
956 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
957 }
960 VOID
961 XenNet_RxResumeEnd(xennet_info_t *xi)
962 {
963 KIRQL old_irql;
965 FUNCTION_ENTER();
967 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
968 //XenNet_BufferAlloc(xi);
969 XenNet_FillRing(xi);
970 KeReleaseSpinLock(&xi->rx_lock, old_irql);
972 FUNCTION_EXIT();
973 }
975 BOOLEAN
976 XenNet_RxInit(xennet_info_t *xi)
977 {
978 NDIS_STATUS status;
980 FUNCTION_ENTER();
982 xi->rx_shutting_down = FALSE;
983 KeInitializeSpinLock(&xi->rx_lock);
984 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
985 KeInitializeTimer(&xi->rx_timer);
986 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
987 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
988 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
989 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
991 XenNet_BufferAlloc(xi);
993 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
994 PROTOCOL_RESERVED_SIZE_IN_PACKET);
995 if (status != NDIS_STATUS_SUCCESS)
996 {
997 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
998 return FALSE;
999 }
1001 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0, LOOKASIDE_LIST_ALLOC_SIZE, XENNET_POOL_TAG, 0);
1003 XenNet_FillRing(xi);
1005 FUNCTION_EXIT();
1007 return TRUE;
1010 BOOLEAN
1011 XenNet_RxShutdown(xennet_info_t *xi)
1013 KIRQL old_irql;
1015 FUNCTION_ENTER();
1017 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1018 xi->rx_shutting_down = TRUE;
1019 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1021 if (xi->config_rx_interrupt_moderation)
1023 KeCancelTimer(&xi->rx_timer);
1026 KeRemoveQueueDpc(&xi->rx_dpc);
1027 KeFlushQueuedDpcs();
1029 while (xi->rx_outstanding)
1031 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1032 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1035 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1037 XenNet_BufferFree(xi);
1039 packet_freelist_dispose(xi);
1041 NdisFreePacketPool(xi->rx_packet_pool);
1043 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1045 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1047 FUNCTION_EXIT();
1049 return TRUE;