win-pvdrivers

view xennet/xennet_rx.c @ 639:16108e228997

Added tag 0.10.0.89 for changeset 70c3a7839b4e
author James Harper <james.harper@bendigoit.com.au>
date Sun Aug 23 14:19:50 2009 +1000 (2009-08-23)
parents ca22d5c09eed
children 7232a030f662
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 static __inline shared_buffer_t *
24 get_pb_from_freelist(struct xennet_info *xi)
25 {
26 shared_buffer_t *pb;
28 if (xi->rx_pb_free == 0)
29 {
30 KdPrint((__DRIVER_NAME " Out of pb's\n"));
31 return NULL;
32 }
33 xi->rx_pb_free--;
35 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
36 pb->ref_count++;
37 return pb;
38 }
40 static __inline VOID
41 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
42 {
43 UNREFERENCED_PARAMETER(xi);
44 pb->ref_count++;
45 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
46 }
48 static __inline VOID
49 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
50 {
51 pb->ref_count--;
52 //KdPrint((__DRIVER_NAME " decremented pb %p ref to %d\n", pb, pb->ref_count));
53 if (pb->ref_count == 0)
54 {
55 //KdPrint((__DRIVER_NAME " freeing pb %p\n", pb, pb->ref_count));
56 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
57 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
58 pb->next = NULL;
59 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
60 xi->rx_pb_free++;
61 }
62 }
64 // Called at DISPATCH_LEVEL with rx lock held
65 static NDIS_STATUS
66 XenNet_FillRing(struct xennet_info *xi)
67 {
68 unsigned short id;
69 shared_buffer_t *page_buf;
70 ULONG i, notify;
71 ULONG batch_target;
72 RING_IDX req_prod = xi->rx.req_prod_pvt;
73 netif_rx_request_t *req;
75 //FUNCTION_ENTER();
77 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
79 if (batch_target < (xi->rx_target >> 2))
80 {
81 //FUNCTION_EXIT();
82 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
83 }
85 for (i = 0; i < batch_target; i++)
86 {
87 page_buf = get_pb_from_freelist(xi);
88 if (!page_buf)
89 {
90 KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
91 break;
92 }
93 xi->rx_id_free--;
95 /* Give to netback */
96 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
97 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
98 xi->rx_ring_pbs[id] = page_buf->id;
99 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
100 req->id = id;
101 req->gref = (grant_ref_t)(page_buf->logical.QuadPart >> PAGE_SHIFT);
102 ASSERT(req->gref != INVALID_GRANT_REF);
103 }
104 KeMemoryBarrier();
105 xi->rx.req_prod_pvt = req_prod + i;
106 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
107 if (notify)
108 {
109 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
110 }
112 //FUNCTION_EXIT();
114 return NDIS_STATUS_SUCCESS;
115 }
117 static PNDIS_PACKET
118 get_packet_from_freelist(struct xennet_info *xi)
119 {
120 NDIS_STATUS status;
121 PNDIS_PACKET packet;
123 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
125 if (!xi->rx_packet_free)
126 {
127 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
128 if (status != NDIS_STATUS_SUCCESS)
129 {
130 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
131 return NULL;
132 }
133 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
134 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
135 }
136 else
137 {
138 xi->rx_packet_free--;
139 packet = xi->rx_packet_list[xi->rx_packet_free];
140 }
141 return packet;
142 }
144 static VOID
145 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
146 {
147 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
148 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
150 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
151 {
152 KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
153 NdisFreePacket(packet);
154 return;
155 }
156 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
157 packet, TcpIpChecksumPacketInfo);
158 csum_info->Value = 0;
159 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
160 xi->rx_packet_list[xi->rx_packet_free] = packet;
161 xi->rx_packet_free++;
162 }
164 static VOID
165 packet_freelist_dispose(struct xennet_info *xi)
166 {
167 while(xi->rx_packet_free != 0)
168 {
169 xi->rx_packet_free--;
170 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
171 }
172 }
174 static PNDIS_PACKET
175 XenNet_MakePacket(struct xennet_info *xi)
176 {
177 NDIS_STATUS status;
178 PNDIS_PACKET packet;
179 PNDIS_BUFFER out_buffer;
180 USHORT new_ip4_length;
181 PUCHAR header_va;
182 packet_info_t *pi = &xi->rxpi;
184 //FUNCTION_ENTER();
186 if (!pi->split_required)
187 {
188 PNDIS_BUFFER buffer;
189 shared_buffer_t *page_buf;
191 //KdPrint((__DRIVER_NAME " !split_required\n"));
193 packet = get_packet_from_freelist(xi);
194 if (packet == NULL)
195 {
196 /* buffers will be freed in MakePackets */
197 KdPrint((__DRIVER_NAME " No free packets\n"));
198 //FUNCTION_EXIT();
199 return NULL;
200 }
201 xi->rx_outstanding++;
203 // what if we needed to consolidate the header? maybe should put that on instead of all the buffers...
204 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = pi->first_pb;
206 buffer = pi->first_buffer;
207 page_buf = pi->first_pb;
208 //KdPrint((__DRIVER_NAME " packet = %p, first_buffer = %p, first_pb = %p\n", packet, buffer, page_buf));
209 while (buffer)
210 {
211 PNDIS_BUFFER next_buffer;
212 //KdPrint((__DRIVER_NAME " buffer = %p\n", buffer));
214 NdisGetNextBuffer(buffer, &next_buffer);
215 NDIS_BUFFER_LINKAGE(buffer) = NULL;
216 NdisChainBufferAtBack(packet, buffer);
217 //ref_pb(xi, page_buf);
219 buffer = next_buffer;
220 page_buf = page_buf->next;
221 }
223 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
224 }
225 else
226 {
227 ULONG out_remaining;
228 shared_buffer_t *header_buf;
230 //KdPrint((__DRIVER_NAME " split_required\n"));
232 packet = get_packet_from_freelist(xi);
233 if (packet == NULL)
234 {
235 //FUNCTION_EXIT();
236 return NULL;
237 }
238 xi->rx_outstanding++;
240 //status = NdisAllocateMemoryWithTag((PUCHAR *)&header_buf, sizeof(shared_buffer_t) + pi->header_length, XENNET_POOL_TAG);
241 ASSERT(sizeof(shared_buffer_t) + pi->header_length < LOOKASIDE_LIST_ALLOC_SIZE);
242 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
243 ASSERT(header_buf); // lazy
244 header_va = (PUCHAR)(header_buf + 1);
245 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
246 NdisMoveMemory(header_va, pi->header, pi->header_length);
248 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too
250 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
251 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
252 NdisChainBufferAtBack(packet, out_buffer);
253 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = header_buf;
254 header_buf->next = pi->curr_pb;
256 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
257 //KdPrint((__DRIVER_NAME " curr_mdl_offset = %d\n", pi->curr_mdl_offset));
258 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
260 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
261 out_remaining = (USHORT)min(pi->mss, pi->tcp_remaining);
262 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + out_remaining);
263 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
264 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
265 pi->tcp_seq += out_remaining;
266 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - out_remaining);
267 do
268 {
269 ULONG in_buffer_offset;
270 ULONG in_buffer_length;
271 ULONG out_length;
272 //UINT tmp_packet_length;
273 //KdPrint((__DRIVER_NAME " curr_buffer = %p\n", pi->curr_buffer));
274 //KdPrint((__DRIVER_NAME " curr_pb = %p\n", pi->curr_pb));
275 //KdPrint((__DRIVER_NAME " out_remaining = %d\n", out_remaining));
276 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
277 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
278 //KdPrint((__DRIVER_NAME " out_length = %d\n", out_length));
279 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
280 //TODO: check status
281 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
282 NdisChainBufferAtBack(packet, out_buffer);
283 //NdisQueryPacketLength(packet, &tmp_packet_length);
284 //KdPrint((__DRIVER_NAME " current packet length = %d\n", tmp_packet_length));
285 ref_pb(xi, pi->curr_pb);
286 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
287 if (pi->curr_mdl_offset == in_buffer_length)
288 {
289 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
290 pi->curr_pb = pi->curr_pb->next;
291 pi->curr_mdl_offset = 0;
292 }
293 out_remaining -= out_length;
294 } while (out_remaining != 0);
295 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
296 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
297 }
299 //FUNCTION_EXIT();
300 return packet;
301 }
303 /*
304 Windows appears to insist that the checksum on received packets is correct, and won't
305 believe us when we lie about it, which happens when the packet is generated on the
306 same bridge in Dom0. Doh!
307 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
308 */
310 static BOOLEAN
311 XenNet_SumPacketData(
312 packet_info_t *pi,
313 PNDIS_PACKET packet,
314 BOOLEAN set_csum
315 )
316 {
317 USHORT i;
318 PUCHAR buffer;
319 PMDL mdl;
320 UINT total_length;
321 UINT buffer_length;
322 USHORT buffer_offset;
323 ULONG csum;
324 PUSHORT csum_ptr;
325 USHORT remaining;
326 USHORT ip4_length;
328 //FUNCTION_ENTER();
330 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
331 ASSERT(mdl);
333 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
335 if ((USHORT)(ip4_length + XN_HDR_SIZE) != total_length)
336 {
337 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
338 }
340 switch (pi->ip_proto)
341 {
342 case 6:
343 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
344 break;
345 case 17:
346 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
347 break;
348 default:
349 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
350 //FUNCTION_EXIT();
351 return FALSE; // should never happen
352 }
354 if (set_csum)
355 *csum_ptr = 0;
357 csum = 0;
358 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
359 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
360 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
362 remaining = ip4_length - pi->ip4_header_length;
364 csum += remaining;
366 for (buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length; i < total_length - 1; i += 2, buffer_offset += 2)
367 {
368 /* don't include the checksum field itself in the calculation */
369 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
370 continue;
371 if (buffer_offset == buffer_length - 1) // deal with a buffer ending on an odd byte boundary
372 {
373 csum += (USHORT)buffer[buffer_offset] << 8;
374 NdisGetNextBuffer(mdl, &mdl);
375 if (mdl == NULL)
376 {
377 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
378 return FALSE; // should never happen
379 }
380 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
381 csum += ((USHORT)buffer[0]);
382 buffer_offset = (USHORT)-1;
383 }
384 else
385 {
386 if (buffer_offset == buffer_length)
387 {
388 // KdPrint((__DRIVER_NAME " New buffer - aligned...\n"));
389 NdisGetNextBuffer(mdl, &mdl);
390 if (mdl == NULL)
391 {
392 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
393 return FALSE; // should never happen
394 }
395 NdisQueryBufferSafe(mdl, (PVOID) &buffer, &buffer_length, NormalPagePriority);
396 buffer_offset = 0;
397 }
398 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
399 }
400 }
401 if (i != total_length) // last odd byte
402 {
403 if (buffer_offset >= buffer_length)
404 {
405 NdisGetNextBuffer(mdl, &mdl);
406 if (mdl == NULL)
407 {
408 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
409 return FALSE; // should never happen
410 }
411 NdisQueryBufferSafe(mdl, (PVOID)&buffer, &buffer_length, NormalPagePriority);
412 buffer_offset = 0;
413 }
414 csum += ((USHORT)buffer[buffer_offset] << 8);
415 }
416 while (csum & 0xFFFF0000)
417 csum = (csum & 0xFFFF) + (csum >> 16);
419 if (set_csum)
420 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
421 else
422 {
423 //FUNCTION_EXIT();
424 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
425 }
426 //FUNCTION_EXIT();
427 return TRUE;
428 }
430 static ULONG
431 XenNet_MakePackets(
432 struct xennet_info *xi,
433 PLIST_ENTRY rx_packet_list
434 )
435 {
436 USHORT i;
437 ULONG packet_count = 0;
438 PNDIS_PACKET packet;
439 PLIST_ENTRY entry;
440 UCHAR psh;
441 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
442 ULONG parse_result;
443 packet_info_t *pi = &xi->rxpi;
444 PNDIS_BUFFER buffer;
445 shared_buffer_t *page_buf;
447 //FUNCTION_ENTER();
449 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
451 //KdPrint((__DRIVER_NAME " ip4_length = %d, tcp_length = %d\n", pi->ip4_length, pi->tcp_length));
453 if ((xi->packet_filter & NDIS_PACKET_TYPE_MULTICAST)
454 && !(xi->packet_filter & NDIS_PACKET_TYPE_ALL_MULTICAST)
455 && (pi->header[0] & 0x01)
456 && !(pi->header[0] == 0xFF && pi->header[1] == 0xFF && pi->header[2] == 0xFF
457 && pi->header[3] == 0xFF && pi->header[4] == 0xFF && pi->header[5] == 0xFF))
458 {
459 for (i = 0; i < xi->multicast_list_size; i++)
460 {
461 if (memcmp(xi->multicast_list[i], pi->header, 6) == 0)
462 break;
463 }
464 if (i == xi->multicast_list_size)
465 {
466 //KdPrint((__DRIVER_NAME " Packet not for my MAC address\n"));
467 goto done;
468 }
469 }
470 switch (pi->ip_proto)
471 {
472 case 6: // TCP
473 if (pi->split_required)
474 break;
475 // fallthrough
476 case 17: // UDP
477 packet = XenNet_MakePacket(xi);
478 if (packet == NULL)
479 {
480 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
481 xi->stat_rx_no_buffer++;
482 packet_count = 0;
483 goto done;
484 }
486 if (parse_result == PARSE_OK)
487 {
488 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
489 packet, TcpIpChecksumPacketInfo);
490 if (pi->csum_blank || pi->data_validated)
491 {
492 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
493 {
494 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
495 {
496 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
497 }
498 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
499 {
500 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
501 }
502 if (pi->csum_blank)
503 {
504 XenNet_SumPacketData(pi, packet, TRUE);
505 }
506 }
507 else if (xi->config_csum_rx_check)
508 {
509 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
510 {
511 if (XenNet_SumPacketData(pi, packet, FALSE))
512 {
513 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
514 }
515 else
516 {
517 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
518 }
519 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
520 {
521 if (XenNet_SumPacketData(pi, packet, FALSE))
522 {
523 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
524 }
525 else
526 {
527 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
528 }
529 }
530 }
531 }
532 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
533 InsertTailList(rx_packet_list, entry);
534 XenNet_ClearPacketInfo(pi);
535 //FUNCTION_EXIT();
536 return 1;
537 default:
538 packet = XenNet_MakePacket(xi);
539 if (packet == NULL)
540 {
541 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
542 xi->stat_rx_no_buffer++;
543 packet_count = 0;
544 goto done;
545 }
546 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
547 InsertTailList(rx_packet_list, entry);
548 XenNet_ClearPacketInfo(pi);
549 //FUNCTION_EXIT();
550 return 1;
551 }
553 /* synchronise the pb with the buffer */
554 buffer = pi->first_buffer;
555 pi->curr_pb = pi->first_pb;
556 //KdPrint((__DRIVER_NAME " first_buffer = %p, curr_buffer = %p, first_pb = %p, curr_pb = %p\n",
557 // pi->first_buffer, pi->curr_buffer, pi->first_pb, pi->curr_pb));
559 while (pi->curr_pb != NULL && buffer != pi->curr_buffer)
560 {
561 NdisGetNextBuffer(buffer, &buffer);
562 pi->curr_pb = pi->curr_pb->next;
563 //KdPrint((__DRIVER_NAME " buffer = %p, curr_pb = %p\n", buffer, pi->curr_pb));
564 }
566 pi->tcp_remaining = pi->tcp_length;
568 /* we can make certain assumptions here as the following code is only for tcp4 */
569 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
570 while (pi->tcp_remaining)
571 {
572 PUCHAR header_va;
573 PMDL mdl;
574 UINT total_length;
575 UINT buffer_length;
576 packet = XenNet_MakePacket(xi);
577 if (!packet)
578 {
579 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
580 xi->stat_rx_no_buffer++;
581 break; /* we are out of memory - just drop the packets */
582 }
583 if (xi->setting_csum.V4Receive.TcpChecksum)
584 {
585 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
586 packet, TcpIpChecksumPacketInfo);
587 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
588 }
589 if (psh)
590 {
591 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
592 if (pi->tcp_remaining)
593 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
594 else
595 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
596 }
597 XenNet_SumPacketData(pi, packet, TRUE);
598 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
599 InsertTailList(rx_packet_list, entry);
600 packet_count++;
601 }
603 done:
604 //buffer = pi->first_buffer;
605 page_buf = pi->first_pb;
606 while (page_buf)
607 {
608 //PNDIS_BUFFER next_buffer;
609 shared_buffer_t *next_pb;
611 //NdisGetNextBuffer(buffer, &next_buffer);
612 next_pb = page_buf->next;
614 //NdisFreeBuffer(buffer);
615 put_pb_on_freelist(xi, page_buf);
617 //buffer = next_buffer;
618 page_buf = next_pb;
619 }
620 XenNet_ClearPacketInfo(pi);
621 //FUNCTION_EXIT();
622 return packet_count;
623 }
625 static BOOLEAN
626 XenNet_RxQueueDpcSynchronized(PVOID context)
627 {
628 struct xennet_info *xi = context;
630 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
632 return TRUE;
633 }
635 #define MAXIMUM_PACKETS_PER_INDICATE 32
636 /*
637 We limit the number of packets per interrupt so that acks get a chance
638 under high rx load. The DPC is immediately re-scheduled */
640 #define MAX_PACKETS_PER_INTERRUPT 64
642 // Called at DISPATCH_LEVEL
643 static VOID
644 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
645 {
646 struct xennet_info *xi = context;
647 RING_IDX cons, prod;
648 LIST_ENTRY rx_packet_list;
649 PLIST_ENTRY entry;
650 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
651 ULONG packet_count = 0;
652 struct netif_rx_response *rxrsp = NULL;
653 struct netif_extra_info *ei;
654 USHORT id;
655 int more_to_do = FALSE;
656 packet_info_t *pi = &xi->rxpi;
657 //NDIS_STATUS status;
658 shared_buffer_t *page_buf;
659 PNDIS_BUFFER buffer;
661 UNREFERENCED_PARAMETER(dpc);
662 UNREFERENCED_PARAMETER(arg1);
663 UNREFERENCED_PARAMETER(arg2);
665 //FUNCTION_ENTER();
667 ASSERT(xi->connected);
669 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
671 if (xi->rx_shutting_down)
672 {
673 /* there is a chance that our Dpc had been queued just before the shutdown... */
674 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
675 return;
676 }
677 InitializeListHead(&rx_packet_list);
679 do {
680 prod = xi->rx.sring->rsp_prod;
681 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
682 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
684 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
685 {
686 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
687 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
688 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
689 xi->rx_ring_pbs[id] = 0xFFFF;
690 xi->rx_id_free++;
691 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
692 if (pi->extra_info)
693 {
694 //KdPrint((__DRIVER_NAME " processing extra info\n"));
695 put_pb_on_freelist(xi, page_buf);
696 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
697 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
698 switch (ei->type)
699 {
700 case XEN_NETIF_EXTRA_TYPE_GSO:
701 switch (ei->u.gso.type)
702 {
703 case XEN_NETIF_GSO_TYPE_TCPV4:
704 pi->mss = ei->u.gso.size;
705 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
706 break;
707 default:
708 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
709 break;
710 }
711 break;
712 default:
713 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
714 break;
715 }
716 }
717 else
718 {
719 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
720 if (rxrsp->status <= 0
721 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
722 {
723 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
724 rxrsp->offset, rxrsp->status));
725 ASSERT(!pi->extra_info);
726 put_pb_on_freelist(xi, page_buf);
727 continue;
728 }
729 ASSERT(!rxrsp->offset);
730 ASSERT(rxrsp->id == id);
731 if (!pi->more_frags) // handling the packet's 1st buffer
732 {
733 if (rxrsp->flags & NETRXF_csum_blank)
734 pi->csum_blank = TRUE;
735 if (rxrsp->flags & NETRXF_data_validated)
736 pi->data_validated = TRUE;
737 }
738 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
739 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
740 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
741 buffer = page_buf->buffer;
742 NdisAdjustBufferLength(buffer, rxrsp->status);
743 if (pi->first_pb)
744 {
745 //KdPrint((__DRIVER_NAME " additional buffer\n"));
746 pi->curr_pb->next = page_buf;
747 pi->curr_pb = page_buf;
748 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
749 pi->curr_buffer = buffer;
750 }
751 else
752 {
753 pi->first_pb = page_buf;
754 pi->curr_pb = page_buf;
755 pi->first_buffer = buffer;
756 pi->curr_buffer = buffer;
757 }
758 pi->mdl_count++;
759 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
760 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
761 pi->total_length = pi->total_length + rxrsp->status;
762 }
764 /* Packet done, add it to the list */
765 if (!pi->more_frags && !pi->extra_info)
766 {
767 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
768 }
769 }
770 xi->rx.rsp_cons = cons;
772 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
773 break;
775 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
776 if (!more_to_do)
777 {
778 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
779 mb();
780 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
781 }
782 } while (more_to_do);
784 if (pi->more_frags || pi->extra_info)
785 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
787 /* Give netback more buffers */
788 XenNet_FillRing(xi);
790 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
791 {
792 /* fire again immediately */
793 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
794 }
796 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
798 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
800 entry = RemoveHeadList(&rx_packet_list);
801 packet_count = 0;
802 while (entry != &rx_packet_list)
803 {
804 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[0]);
805 packets[packet_count++] = packet;
806 entry = RemoveHeadList(&rx_packet_list);
807 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
808 {
809 //KdPrint((__DRIVER_NAME " Indicating\n"));
810 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
811 packet_count = 0;
812 }
813 }
814 //FUNCTION_EXIT();
815 }
817 /* called at DISPATCH_LEVEL */
818 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
819 VOID DDKAPI
820 XenNet_ReturnPacket(
821 IN NDIS_HANDLE MiniportAdapterContext,
822 IN PNDIS_PACKET Packet
823 )
824 {
825 struct xennet_info *xi = MiniportAdapterContext;
826 PNDIS_BUFFER buffer;
827 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[sizeof(LIST_ENTRY)];
829 //FUNCTION_ENTER();
831 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
833 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
835 NdisUnchainBufferAtFront(Packet, &buffer);
836 while (buffer)
837 {
838 shared_buffer_t *next_buf = page_buf->next;
839 if (!page_buf->virtual)
840 {
841 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
842 PUCHAR va;
843 UINT len;
844 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
845 //KdPrint((__DRIVER_NAME " freeing header buffer %p\n", va - sizeof(shared_buffer_t)));
846 //NdisFreeMemory(va - sizeof(shared_buffer_t), len + sizeof(shared_buffer_t), 0);
847 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
848 NdisFreeBuffer(buffer);
849 }
850 else
851 {
852 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
853 if (buffer != page_buf->buffer)
854 NdisFreeBuffer(buffer);
855 put_pb_on_freelist(xi, page_buf);
856 }
857 NdisUnchainBufferAtFront(Packet, &buffer);
858 page_buf = next_buf;
859 }
861 put_packet_on_freelist(xi, Packet);
862 xi->rx_outstanding--;
864 if (!xi->rx_outstanding && xi->rx_shutting_down)
865 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
867 XenNet_FillRing(xi);
869 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
871 //FUNCTION_EXIT();
872 }
874 /*
875 Free all Rx buffers (on halt, for example)
876 The ring must be stopped at this point.
877 */
879 static VOID
880 XenNet_PurgeRing(struct xennet_info *xi)
881 {
882 int i;
883 for (i = 0; i < NET_RX_RING_SIZE; i++)
884 {
885 if (xi->rx_ring_pbs[i] != 0xFFFF)
886 {
887 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
888 xi->rx_ring_pbs[i] = 0xFFFF;
889 }
890 }
891 }
893 static VOID
894 XenNet_BufferFree(struct xennet_info *xi)
895 {
896 shared_buffer_t *pb;
898 XenNet_PurgeRing(xi);
900 while ((pb = get_pb_from_freelist(xi)) != NULL)
901 {
902 NdisFreeBuffer(pb->buffer);
903 NdisMFreeSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, pb->virtual, pb->logical);
904 }
905 }
907 VOID
908 XenNet_RxResumeStart(xennet_info_t *xi)
909 {
910 KIRQL old_irql;
912 FUNCTION_ENTER();
914 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
915 XenNet_PurgeRing(xi);
916 KeReleaseSpinLock(&xi->rx_lock, old_irql);
918 FUNCTION_EXIT();
919 }
921 VOID
922 XenNet_BufferAlloc(xennet_info_t *xi)
923 {
924 NDIS_STATUS status;
925 int i;
927 xi->rx_id_free = NET_RX_RING_SIZE;
928 xi->rx_outstanding = 0;
930 for (i = 0; i < NET_RX_RING_SIZE; i++)
931 {
932 xi->rx_ring_pbs[i] = 0xFFFF;
933 }
935 for (i = 0; i < RX_PAGE_BUFFERS; i++)
936 {
937 xi->rx_pbs[i].id = (USHORT)i;
938 NdisMAllocateSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, &xi->rx_pbs[i].virtual, &xi->rx_pbs[i].logical);
939 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
940 if (status != STATUS_SUCCESS)
941 break;
942 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
943 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
944 }
945 if (i == 0)
946 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
947 }
950 VOID
951 XenNet_RxResumeEnd(xennet_info_t *xi)
952 {
953 KIRQL old_irql;
955 FUNCTION_ENTER();
957 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
958 //XenNet_BufferAlloc(xi);
959 XenNet_FillRing(xi);
960 KeReleaseSpinLock(&xi->rx_lock, old_irql);
962 FUNCTION_EXIT();
963 }
965 BOOLEAN
966 XenNet_RxInit(xennet_info_t *xi)
967 {
968 NDIS_STATUS status;
970 FUNCTION_ENTER();
972 xi->rx_shutting_down = FALSE;
973 KeInitializeSpinLock(&xi->rx_lock);
974 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
975 KeInitializeTimer(&xi->rx_timer);
976 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
977 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
978 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
979 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
981 XenNet_BufferAlloc(xi);
983 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
984 PROTOCOL_RESERVED_SIZE_IN_PACKET);
985 if (status != NDIS_STATUS_SUCCESS)
986 {
987 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
988 return FALSE;
989 }
991 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0, LOOKASIDE_LIST_ALLOC_SIZE, XENNET_POOL_TAG, 0);
993 XenNet_FillRing(xi);
995 FUNCTION_EXIT();
997 return TRUE;
998 }
1000 BOOLEAN
1001 XenNet_RxShutdown(xennet_info_t *xi)
1003 KIRQL old_irql;
1005 FUNCTION_ENTER();
1007 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1008 xi->rx_shutting_down = TRUE;
1009 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1011 if (xi->config_rx_interrupt_moderation)
1013 KeCancelTimer(&xi->rx_timer);
1016 KeRemoveQueueDpc(&xi->rx_dpc);
1017 KeFlushQueuedDpcs();
1019 while (xi->rx_outstanding)
1021 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1022 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1025 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1027 XenNet_BufferFree(xi);
1029 packet_freelist_dispose(xi);
1031 NdisFreePacketPool(xi->rx_packet_pool);
1033 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1035 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1037 FUNCTION_EXIT();
1039 return TRUE;