win-pvdrivers

view xennet/xennet_rx.c @ 573:3e36937b0bf5

fixed the inverted meaning of the config_csum_rx_check setting
author James Harper <james.harper@bendigoit.com.au>
date Wed May 20 21:39:57 2009 +1000 (2009-05-20)
parents b0b8be2d30c0
children dc0a293c870c
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 static __inline shared_buffer_t *
24 get_pb_from_freelist(struct xennet_info *xi)
25 {
26 shared_buffer_t *pb;
28 if (xi->rx_pb_free == 0)
29 {
30 KdPrint((__DRIVER_NAME " Out of pb's\n"));
31 return NULL;
32 }
33 xi->rx_pb_free--;
35 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
36 pb->ref_count++;
37 return pb;
38 }
40 static __inline VOID
41 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
42 {
43 UNREFERENCED_PARAMETER(xi);
44 pb->ref_count++;
45 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
46 }
48 static __inline VOID
49 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
50 {
51 pb->ref_count--;
52 //KdPrint((__DRIVER_NAME " decremented pb %p ref to %d\n", pb, pb->ref_count));
53 if (pb->ref_count == 0)
54 {
55 //KdPrint((__DRIVER_NAME " freeing pb %p\n", pb, pb->ref_count));
56 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
57 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
58 pb->next = NULL;
59 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
60 xi->rx_pb_free++;
61 }
62 }
64 // Called at DISPATCH_LEVEL with rx lock held
65 static NDIS_STATUS
66 XenNet_FillRing(struct xennet_info *xi)
67 {
68 unsigned short id;
69 shared_buffer_t *page_buf;
70 ULONG i, notify;
71 ULONG batch_target;
72 RING_IDX req_prod = xi->rx.req_prod_pvt;
73 netif_rx_request_t *req;
75 //FUNCTION_ENTER();
77 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
79 if (batch_target < (xi->rx_target >> 2))
80 {
81 //FUNCTION_EXIT();
82 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
83 }
85 for (i = 0; i < batch_target; i++)
86 {
87 page_buf = get_pb_from_freelist(xi);
88 if (!page_buf)
89 {
90 KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
91 break;
92 }
93 xi->rx_id_free--;
95 /* Give to netback */
96 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
97 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
98 xi->rx_ring_pbs[id] = page_buf->id;
99 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
100 req->id = id;
101 req->gref = (grant_ref_t)(page_buf->logical.QuadPart >> PAGE_SHIFT);
102 ASSERT(req->gref != INVALID_GRANT_REF);
103 }
104 KeMemoryBarrier();
105 xi->rx.req_prod_pvt = req_prod + i;
106 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
107 if (notify)
108 {
109 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
110 }
112 //FUNCTION_EXIT();
114 return NDIS_STATUS_SUCCESS;
115 }
117 static PNDIS_PACKET
118 get_packet_from_freelist(struct xennet_info *xi)
119 {
120 NDIS_STATUS status;
121 PNDIS_PACKET packet;
123 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
125 if (!xi->rx_packet_free)
126 {
127 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
128 if (status != NDIS_STATUS_SUCCESS)
129 {
130 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
131 return NULL;
132 }
133 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
134 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
135 }
136 else
137 {
138 xi->rx_packet_free--;
139 packet = xi->rx_packet_list[xi->rx_packet_free];
140 }
141 return packet;
142 }
144 static VOID
145 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
146 {
147 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
148 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
150 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
151 {
152 KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
153 NdisFreePacket(packet);
154 return;
155 }
156 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
157 packet, TcpIpChecksumPacketInfo);
158 csum_info->Value = 0;
159 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
160 xi->rx_packet_list[xi->rx_packet_free] = packet;
161 xi->rx_packet_free++;
162 }
164 static VOID
165 packet_freelist_dispose(struct xennet_info *xi)
166 {
167 while(xi->rx_packet_free != 0)
168 {
169 xi->rx_packet_free--;
170 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
171 }
172 }
174 static PNDIS_PACKET
175 XenNet_MakePacket(struct xennet_info *xi)
176 {
177 NDIS_STATUS status;
178 PNDIS_PACKET packet;
179 PNDIS_BUFFER out_buffer;
180 USHORT new_ip4_length;
181 PUCHAR header_va;
182 packet_info_t *pi = &xi->rxpi;
184 //FUNCTION_ENTER();
186 if (!pi->split_required)
187 {
188 PNDIS_BUFFER buffer;
189 shared_buffer_t *page_buf;
191 //KdPrint((__DRIVER_NAME " !split_required\n"));
193 packet = get_packet_from_freelist(xi);
194 if (packet == NULL)
195 {
196 /* buffers will be freed in MakePackets */
197 KdPrint((__DRIVER_NAME " No free packets\n"));
198 //FUNCTION_EXIT();
199 return NULL;
200 }
201 xi->rx_outstanding++;
203 // what if we needed to consolidate the header? maybe should put that on instead of all the buffers...
204 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = pi->first_pb;
206 buffer = pi->first_buffer;
207 page_buf = pi->first_pb;
208 //KdPrint((__DRIVER_NAME " packet = %p, first_buffer = %p, first_pb = %p\n", packet, buffer, page_buf));
209 while (buffer)
210 {
211 PNDIS_BUFFER next_buffer;
212 //KdPrint((__DRIVER_NAME " buffer = %p\n", buffer));
214 NdisGetNextBuffer(buffer, &next_buffer);
215 NDIS_BUFFER_LINKAGE(buffer) = NULL;
216 NdisChainBufferAtBack(packet, buffer);
217 //ref_pb(xi, page_buf);
219 buffer = next_buffer;
220 page_buf = page_buf->next;
221 }
223 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
224 }
225 else
226 {
227 ULONG out_remaining;
228 shared_buffer_t *header_buf;
230 //KdPrint((__DRIVER_NAME " split_required\n"));
232 packet = get_packet_from_freelist(xi);
233 if (packet == NULL)
234 {
235 //FUNCTION_EXIT();
236 return NULL;
237 }
238 xi->rx_outstanding++;
240 //status = NdisAllocateMemoryWithTag((PUCHAR *)&header_buf, sizeof(shared_buffer_t) + pi->header_length, XENNET_POOL_TAG);
241 ASSERT(sizeof(shared_buffer_t) + pi->header_length < LOOKASIDE_LIST_ALLOC_SIZE);
242 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
243 ASSERT(header_buf); // lazy
244 header_va = (PUCHAR)(header_buf + 1);
245 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
246 NdisMoveMemory(header_va, pi->header, pi->header_length);
248 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too
250 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
251 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
252 NdisChainBufferAtBack(packet, out_buffer);
253 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = header_buf;
254 header_buf->next = pi->curr_pb;
256 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
257 //KdPrint((__DRIVER_NAME " curr_mdl_offset = %d\n", pi->curr_mdl_offset));
258 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
260 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
261 out_remaining = (USHORT)min(pi->mss, pi->tcp_remaining);
262 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + out_remaining);
263 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
264 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
265 pi->tcp_seq += out_remaining;
266 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - out_remaining);
267 do
268 {
269 ULONG in_buffer_offset;
270 ULONG in_buffer_length;
271 ULONG out_length;
272 //UINT tmp_packet_length;
273 //KdPrint((__DRIVER_NAME " curr_buffer = %p\n", pi->curr_buffer));
274 //KdPrint((__DRIVER_NAME " curr_pb = %p\n", pi->curr_pb));
275 //KdPrint((__DRIVER_NAME " out_remaining = %d\n", out_remaining));
276 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
277 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
278 //KdPrint((__DRIVER_NAME " out_length = %d\n", out_length));
279 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
280 //TODO: check status
281 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
282 NdisChainBufferAtBack(packet, out_buffer);
283 //NdisQueryPacketLength(packet, &tmp_packet_length);
284 //KdPrint((__DRIVER_NAME " current packet length = %d\n", tmp_packet_length));
285 ref_pb(xi, pi->curr_pb);
286 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
287 if (pi->curr_mdl_offset == in_buffer_length)
288 {
289 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
290 pi->curr_pb = pi->curr_pb->next;
291 pi->curr_mdl_offset = 0;
292 }
293 out_remaining -= out_length;
294 } while (out_remaining != 0);
295 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
296 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
297 }
299 //FUNCTION_EXIT();
300 return packet;
301 }
303 /*
304 Windows appears to insist that the checksum on received packets is correct, and won't
305 believe us when we lie about it, which happens when the packet is generated on the
306 same bridge in Dom0. Doh!
307 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
308 */
310 static BOOLEAN
311 XenNet_SumPacketData(
312 packet_info_t *pi,
313 PNDIS_PACKET packet,
314 BOOLEAN set_csum
315 )
316 {
317 USHORT i;
318 PUCHAR buffer;
319 PMDL mdl;
320 UINT total_length;
321 UINT buffer_length;
322 USHORT buffer_offset;
323 ULONG csum;
324 PUSHORT csum_ptr;
325 USHORT remaining;
326 USHORT ip4_length;
328 //FUNCTION_ENTER();
330 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
331 ASSERT(mdl);
333 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
335 if ((USHORT)(ip4_length + XN_HDR_SIZE) != total_length)
336 {
337 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
338 }
340 switch (pi->ip_proto)
341 {
342 case 6:
343 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
344 break;
345 case 17:
346 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
347 break;
348 default:
349 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
350 //FUNCTION_EXIT();
351 return FALSE; // should never happen
352 }
354 if (set_csum)
355 *csum_ptr = 0;
357 csum = 0;
358 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
359 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
360 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
362 remaining = ip4_length - pi->ip4_header_length;
364 csum += remaining;
366 for (buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length; i < total_length - 1; i += 2, buffer_offset += 2)
367 {
368 /* don't include the checksum field itself in the calculation */
369 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
370 continue;
371 if (buffer_offset == buffer_length - 1) // deal with a buffer ending on an odd byte boundary
372 {
373 csum += (USHORT)buffer[buffer_offset] << 8;
374 NdisGetNextBuffer(mdl, &mdl);
375 if (mdl == NULL)
376 {
377 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
378 return FALSE; // should never happen
379 }
380 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
381 csum += ((USHORT)buffer[0]);
382 buffer_offset = (USHORT)-1;
383 }
384 else
385 {
386 if (buffer_offset == buffer_length)
387 {
388 // KdPrint((__DRIVER_NAME " New buffer - aligned...\n"));
389 NdisGetNextBuffer(mdl, &mdl);
390 if (mdl == NULL)
391 {
392 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
393 return FALSE; // should never happen
394 }
395 NdisQueryBufferSafe(mdl, (PVOID) &buffer, &buffer_length, NormalPagePriority);
396 buffer_offset = 0;
397 }
398 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
399 }
400 }
401 if (i != total_length) // last odd byte
402 {
403 csum += ((USHORT)buffer[buffer_offset] << 8);
404 }
405 while (csum & 0xFFFF0000)
406 csum = (csum & 0xFFFF) + (csum >> 16);
408 if (set_csum)
409 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
410 else
411 {
412 //FUNCTION_EXIT();
413 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
414 }
415 //FUNCTION_EXIT();
416 return TRUE;
417 }
419 static ULONG
420 XenNet_MakePackets(
421 struct xennet_info *xi,
422 PLIST_ENTRY rx_packet_list
423 )
424 {
425 USHORT i;
426 ULONG packet_count = 0;
427 PNDIS_PACKET packet;
428 PLIST_ENTRY entry;
429 UCHAR psh;
430 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
431 ULONG parse_result;
432 packet_info_t *pi = &xi->rxpi;
433 PNDIS_BUFFER buffer;
434 shared_buffer_t *page_buf;
436 //FUNCTION_ENTER();
438 parse_result = XenNet_ParsePacketHeader(pi);
440 if ((xi->packet_filter & NDIS_PACKET_TYPE_MULTICAST)
441 && !(xi->packet_filter & NDIS_PACKET_TYPE_ALL_MULTICAST)
442 && (pi->header[0] & 0x01)
443 && !(pi->header[0] == 0xFF && pi->header[1] == 0xFF && pi->header[2] == 0xFF
444 && pi->header[3] == 0xFF && pi->header[4] == 0xFF && pi->header[5] == 0xFF))
445 {
446 for (i = 0; i < xi->multicast_list_size; i++)
447 {
448 if (memcmp(xi->multicast_list[i], pi->header, 6) == 0)
449 break;
450 }
451 if (i == xi->multicast_list_size)
452 {
453 //KdPrint((__DRIVER_NAME " Packet not for my MAC address\n"));
454 goto done;
455 }
456 }
457 switch (pi->ip_proto)
458 {
459 case 6: // TCP
460 if (pi->split_required)
461 break;
462 // fallthrough
463 case 17: // UDP
464 packet = XenNet_MakePacket(xi);
465 if (packet == NULL)
466 {
467 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
468 xi->stat_rx_no_buffer++;
469 packet_count = 0;
470 goto done;
471 }
473 if (parse_result == PARSE_OK)
474 {
475 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
476 packet, TcpIpChecksumPacketInfo);
477 if (pi->csum_blank || pi->data_validated)
478 {
479 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
480 {
481 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
482 {
483 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
484 }
485 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
486 {
487 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
488 }
489 if (pi->csum_blank)
490 {
491 XenNet_SumPacketData(pi, packet, TRUE);
492 }
493 }
494 else if (xi->config_csum_rx_check)
495 {
496 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
497 {
498 if (XenNet_SumPacketData(pi, packet, FALSE))
499 {
500 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
501 }
502 else
503 {
504 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
505 }
506 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
507 {
508 if (XenNet_SumPacketData(pi, packet, FALSE))
509 {
510 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
511 }
512 else
513 {
514 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
515 }
516 }
517 }
518 }
519 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
520 InsertTailList(rx_packet_list, entry);
521 XenNet_ClearPacketInfo(pi);
522 //FUNCTION_EXIT();
523 return 1;
524 default:
525 packet = XenNet_MakePacket(xi);
526 if (packet == NULL)
527 {
528 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
529 xi->stat_rx_no_buffer++;
530 packet_count = 0;
531 goto done;
532 }
533 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
534 InsertTailList(rx_packet_list, entry);
535 XenNet_ClearPacketInfo(pi);
536 //FUNCTION_EXIT();
537 return 1;
538 }
540 /* synchronise the pb with the buffer */
541 buffer = pi->first_buffer;
542 pi->curr_pb = pi->first_pb;
543 //KdPrint((__DRIVER_NAME " first_buffer = %p, curr_buffer = %p, first_pb = %p, curr_pb = %p\n",
544 // pi->first_buffer, pi->curr_buffer, pi->first_pb, pi->curr_pb));
546 while (pi->curr_pb != NULL && buffer != pi->curr_buffer)
547 {
548 NdisGetNextBuffer(buffer, &buffer);
549 pi->curr_pb = pi->curr_pb->next;
550 //KdPrint((__DRIVER_NAME " buffer = %p, curr_pb = %p\n", buffer, pi->curr_pb));
551 }
553 pi->tcp_remaining = pi->tcp_length;
555 /* we can make certain assumptions here as the following code is only for tcp4 */
556 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
557 while (pi->tcp_remaining)
558 {
559 PUCHAR header_va;
560 PMDL mdl;
561 UINT total_length;
562 UINT buffer_length;
563 packet = XenNet_MakePacket(xi);
564 if (!packet)
565 {
566 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
567 xi->stat_rx_no_buffer++;
568 break; /* we are out of memory - just drop the packets */
569 }
570 if (xi->setting_csum.V4Receive.TcpChecksum)
571 {
572 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
573 packet, TcpIpChecksumPacketInfo);
574 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
575 }
576 if (psh)
577 {
578 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
579 if (pi->tcp_remaining)
580 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
581 else
582 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
583 }
584 XenNet_SumPacketData(pi, packet, TRUE);
585 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
586 InsertTailList(rx_packet_list, entry);
587 packet_count++;
588 }
590 done:
591 //buffer = pi->first_buffer;
592 page_buf = pi->first_pb;
593 while (page_buf)
594 {
595 //PNDIS_BUFFER next_buffer;
596 shared_buffer_t *next_pb;
598 //NdisGetNextBuffer(buffer, &next_buffer);
599 next_pb = page_buf->next;
601 //NdisFreeBuffer(buffer);
602 put_pb_on_freelist(xi, page_buf);
604 //buffer = next_buffer;
605 page_buf = next_pb;
606 }
607 XenNet_ClearPacketInfo(pi);
608 //FUNCTION_EXIT();
609 return packet_count;
610 }
612 static BOOLEAN
613 XenNet_RxQueueDpcSynchronized(PVOID context)
614 {
615 struct xennet_info *xi = context;
617 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
619 return TRUE;
620 }
622 #define MAXIMUM_PACKETS_PER_INDICATE 32
623 /*
624 We limit the number of packets per interrupt so that acks get a chance
625 under high rx load. The DPC is immediately re-scheduled */
627 #define MAX_PACKETS_PER_INTERRUPT 64
629 // Called at DISPATCH_LEVEL
630 static VOID
631 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
632 {
633 struct xennet_info *xi = context;
634 RING_IDX cons, prod;
635 LIST_ENTRY rx_packet_list;
636 PLIST_ENTRY entry;
637 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
638 ULONG packet_count = 0;
639 struct netif_rx_response *rxrsp = NULL;
640 struct netif_extra_info *ei;
641 USHORT id;
642 int more_to_do = FALSE;
643 packet_info_t *pi = &xi->rxpi;
644 //NDIS_STATUS status;
645 shared_buffer_t *page_buf;
646 PNDIS_BUFFER buffer;
648 UNREFERENCED_PARAMETER(dpc);
649 UNREFERENCED_PARAMETER(arg1);
650 UNREFERENCED_PARAMETER(arg2);
652 //FUNCTION_ENTER();
654 ASSERT(xi->connected);
656 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
658 InitializeListHead(&rx_packet_list);
660 do {
661 prod = xi->rx.sring->rsp_prod;
662 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
663 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
665 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
666 {
667 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
668 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
669 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
670 xi->rx_ring_pbs[id] = 0xFFFF;
671 xi->rx_id_free++;
672 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
673 if (pi->extra_info)
674 {
675 //KdPrint((__DRIVER_NAME " processing extra info\n"));
676 put_pb_on_freelist(xi, page_buf);
677 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
678 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
679 switch (ei->type)
680 {
681 case XEN_NETIF_EXTRA_TYPE_GSO:
682 switch (ei->u.gso.type)
683 {
684 case XEN_NETIF_GSO_TYPE_TCPV4:
685 pi->mss = ei->u.gso.size;
686 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
687 break;
688 default:
689 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
690 break;
691 }
692 break;
693 default:
694 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
695 break;
696 }
697 }
698 else
699 {
700 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
701 //KdPrint((__DRIVER_NAME " status = %d, offset = %d\n", rxrsp->status, rxrsp->offset));
702 if (rxrsp->status <= 0
703 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
704 {
705 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
706 rxrsp->offset, rxrsp->status));
707 ASSERT(!pi->extra_info);
708 put_pb_on_freelist(xi, page_buf);
709 continue;
710 }
711 ASSERT(!rxrsp->offset);
712 ASSERT(rxrsp->id == id);
713 if (!pi->more_frags) // handling the packet's 1st buffer
714 {
715 if (rxrsp->flags & NETRXF_csum_blank)
716 pi->csum_blank = TRUE;
717 if (rxrsp->flags & NETRXF_data_validated)
718 pi->data_validated = TRUE;
719 }
720 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
721 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
722 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
723 buffer = page_buf->buffer;
724 NdisAdjustBufferLength(buffer, rxrsp->status);
725 if (pi->first_pb)
726 {
727 //KdPrint((__DRIVER_NAME " additional buffer\n"));
728 pi->curr_pb->next = page_buf;
729 pi->curr_pb = page_buf;
730 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
731 pi->curr_buffer = buffer;
732 }
733 else
734 {
735 pi->first_pb = page_buf;
736 pi->curr_pb = page_buf;
737 pi->first_buffer = buffer;
738 pi->curr_buffer = buffer;
739 }
740 pi->mdl_count++;
741 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
742 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
743 pi->total_length = pi->total_length + rxrsp->status;
744 }
746 /* Packet done, add it to the list */
747 if (!pi->more_frags && !pi->extra_info)
748 {
749 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
750 }
751 }
752 xi->rx.rsp_cons = cons;
754 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
755 break;
757 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
758 if (!more_to_do)
759 {
760 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
761 mb();
762 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
763 }
764 } while (more_to_do);
766 if (pi->more_frags || pi->extra_info)
767 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
769 /* Give netback more buffers */
770 XenNet_FillRing(xi);
772 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
773 {
774 /* fire again immediately */
775 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
776 }
778 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
780 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
782 entry = RemoveHeadList(&rx_packet_list);
783 packet_count = 0;
784 while (entry != &rx_packet_list)
785 {
786 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[0]);
787 packets[packet_count++] = packet;
788 #if 0
789 KdPrint((__DRIVER_NAME " Adding packet %p to indicate list\n", packet));
790 {
791 PVOID addr;
792 UINT buffer_length;
793 UINT total_length;
794 PNDIS_BUFFER buffer;
796 NdisGetFirstBufferFromPacketSafe(packet, &buffer, &addr, &buffer_length, &total_length, NormalPagePriority);
797 KdPrint((__DRIVER_NAME " buffer = %p, addr = %p, buffer_length = %d, total_length = %d\n", buffer, addr, buffer_length, total_length));
798 ASSERT(total_length <= xi->config_mtu + XN_HDR_SIZE);
799 }
800 #endif
801 entry = RemoveHeadList(&rx_packet_list);
802 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
803 {
804 //KdPrint((__DRIVER_NAME " Indicating\n"));
805 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
806 packet_count = 0;
807 }
808 }
809 //FUNCTION_EXIT();
810 }
812 /* called at DISPATCH_LEVEL */
813 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
814 VOID DDKAPI
815 XenNet_ReturnPacket(
816 IN NDIS_HANDLE MiniportAdapterContext,
817 IN PNDIS_PACKET Packet
818 )
819 {
820 struct xennet_info *xi = MiniportAdapterContext;
821 PNDIS_BUFFER buffer;
822 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[sizeof(LIST_ENTRY)];
824 //FUNCTION_ENTER();
826 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
828 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
830 NdisUnchainBufferAtFront(Packet, &buffer);
831 while (buffer)
832 {
833 shared_buffer_t *next_buf = page_buf->next;
834 if (!page_buf->virtual)
835 {
836 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
837 PUCHAR va;
838 UINT len;
839 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
840 //KdPrint((__DRIVER_NAME " freeing header buffer %p\n", va - sizeof(shared_buffer_t)));
841 //NdisFreeMemory(va - sizeof(shared_buffer_t), len + sizeof(shared_buffer_t), 0);
842 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
843 NdisFreeBuffer(buffer);
844 }
845 else
846 {
847 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
848 if (buffer != page_buf->buffer)
849 NdisFreeBuffer(buffer);
850 put_pb_on_freelist(xi, page_buf);
851 }
852 NdisUnchainBufferAtFront(Packet, &buffer);
853 page_buf = next_buf;
854 }
856 put_packet_on_freelist(xi, Packet);
857 xi->rx_outstanding--;
859 if (!xi->rx_outstanding && xi->rx_shutting_down)
860 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
862 XenNet_FillRing(xi);
864 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
866 //FUNCTION_EXIT();
867 }
869 /*
870 Free all Rx buffers (on halt, for example)
871 The ring must be stopped at this point.
872 */
874 static VOID
875 XenNet_PurgeRing(struct xennet_info *xi)
876 {
877 int i;
878 for (i = 0; i < NET_RX_RING_SIZE; i++)
879 {
880 if (xi->rx_ring_pbs[i] != 0xFFFF)
881 {
882 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
883 xi->rx_ring_pbs[i] = 0xFFFF;
884 }
885 }
886 }
888 static VOID
889 XenNet_BufferFree(struct xennet_info *xi)
890 {
891 shared_buffer_t *pb;
893 XenNet_PurgeRing(xi);
895 while ((pb = get_pb_from_freelist(xi)) != NULL)
896 {
897 NdisFreeBuffer(pb->buffer);
898 NdisMFreeSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, pb->virtual, pb->logical);
899 }
900 }
902 VOID
903 XenNet_RxResumeStart(xennet_info_t *xi)
904 {
905 KIRQL old_irql;
907 FUNCTION_ENTER();
909 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
910 XenNet_PurgeRing(xi);
911 KeReleaseSpinLock(&xi->rx_lock, old_irql);
913 FUNCTION_EXIT();
914 }
916 VOID
917 XenNet_BufferAlloc(xennet_info_t *xi)
918 {
919 NDIS_STATUS status;
920 int i;
922 xi->rx_id_free = NET_RX_RING_SIZE;
923 xi->rx_outstanding = 0;
925 for (i = 0; i < NET_RX_RING_SIZE; i++)
926 {
927 xi->rx_ring_pbs[i] = 0xFFFF;
928 }
930 for (i = 0; i < RX_PAGE_BUFFERS; i++)
931 {
932 xi->rx_pbs[i].id = (USHORT)i;
933 NdisMAllocateSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, &xi->rx_pbs[i].virtual, &xi->rx_pbs[i].logical);
934 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
935 if (status != STATUS_SUCCESS)
936 break;
937 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
938 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
939 }
940 if (i == 0)
941 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
942 }
945 VOID
946 XenNet_RxResumeEnd(xennet_info_t *xi)
947 {
948 KIRQL old_irql;
950 FUNCTION_ENTER();
952 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
953 //XenNet_BufferAlloc(xi);
954 XenNet_FillRing(xi);
955 KeReleaseSpinLock(&xi->rx_lock, old_irql);
957 FUNCTION_EXIT();
958 }
960 BOOLEAN
961 XenNet_RxInit(xennet_info_t *xi)
962 {
963 NDIS_STATUS status;
965 FUNCTION_ENTER();
967 KeInitializeSpinLock(&xi->rx_lock);
968 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
969 KeInitializeTimer(&xi->rx_timer);
970 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
971 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
972 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
973 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
974 xi->rx_shutting_down = FALSE;
976 XenNet_BufferAlloc(xi);
978 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
979 PROTOCOL_RESERVED_SIZE_IN_PACKET);
980 if (status != NDIS_STATUS_SUCCESS)
981 {
982 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
983 return FALSE;
984 }
986 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0, LOOKASIDE_LIST_ALLOC_SIZE, XENNET_POOL_TAG, 0);
988 XenNet_FillRing(xi);
990 FUNCTION_EXIT();
992 return TRUE;
993 }
995 BOOLEAN
996 XenNet_RxShutdown(xennet_info_t *xi)
997 {
998 KIRQL OldIrql;
1000 FUNCTION_ENTER();
1002 KeAcquireSpinLock(&xi->rx_lock, &OldIrql);
1003 xi->rx_shutting_down = TRUE;
1004 KeReleaseSpinLock(&xi->rx_lock, OldIrql);
1006 if (xi->config_rx_interrupt_moderation)
1008 KeCancelTimer(&xi->rx_timer);
1011 while (xi->rx_outstanding)
1013 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1014 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1017 KeAcquireSpinLock(&xi->rx_lock, &OldIrql);
1019 XenNet_BufferFree(xi);
1021 packet_freelist_dispose(xi);
1023 NdisFreePacketPool(xi->rx_packet_pool);
1025 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1027 KeReleaseSpinLock(&xi->rx_lock, OldIrql);
1029 FUNCTION_EXIT();
1031 return TRUE;