win-pvdrivers

view xennet/xennet_rx.c @ 607:c2aea3e060a7

Missed a line in cut&paste in previous checksum fix
author James Harper <james.harper@bendigoit.com.au>
date Sun Jul 05 21:06:26 2009 +1000 (2009-07-05)
parents ee9c59a5a06c
children f5e20f53699d
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 static __inline shared_buffer_t *
24 get_pb_from_freelist(struct xennet_info *xi)
25 {
26 shared_buffer_t *pb;
28 if (xi->rx_pb_free == 0)
29 {
30 KdPrint((__DRIVER_NAME " Out of pb's\n"));
31 return NULL;
32 }
33 xi->rx_pb_free--;
35 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
36 pb->ref_count++;
37 return pb;
38 }
40 static __inline VOID
41 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
42 {
43 UNREFERENCED_PARAMETER(xi);
44 pb->ref_count++;
45 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
46 }
48 static __inline VOID
49 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
50 {
51 pb->ref_count--;
52 //KdPrint((__DRIVER_NAME " decremented pb %p ref to %d\n", pb, pb->ref_count));
53 if (pb->ref_count == 0)
54 {
55 //KdPrint((__DRIVER_NAME " freeing pb %p\n", pb, pb->ref_count));
56 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
57 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
58 pb->next = NULL;
59 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
60 xi->rx_pb_free++;
61 }
62 }
64 // Called at DISPATCH_LEVEL with rx lock held
65 static NDIS_STATUS
66 XenNet_FillRing(struct xennet_info *xi)
67 {
68 unsigned short id;
69 shared_buffer_t *page_buf;
70 ULONG i, notify;
71 ULONG batch_target;
72 RING_IDX req_prod = xi->rx.req_prod_pvt;
73 netif_rx_request_t *req;
75 //FUNCTION_ENTER();
77 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
79 if (batch_target < (xi->rx_target >> 2))
80 {
81 //FUNCTION_EXIT();
82 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
83 }
85 for (i = 0; i < batch_target; i++)
86 {
87 page_buf = get_pb_from_freelist(xi);
88 if (!page_buf)
89 {
90 KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
91 break;
92 }
93 xi->rx_id_free--;
95 /* Give to netback */
96 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
97 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
98 xi->rx_ring_pbs[id] = page_buf->id;
99 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
100 req->id = id;
101 req->gref = (grant_ref_t)(page_buf->logical.QuadPart >> PAGE_SHIFT);
102 ASSERT(req->gref != INVALID_GRANT_REF);
103 }
104 KeMemoryBarrier();
105 xi->rx.req_prod_pvt = req_prod + i;
106 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
107 if (notify)
108 {
109 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
110 }
112 //FUNCTION_EXIT();
114 return NDIS_STATUS_SUCCESS;
115 }
117 static PNDIS_PACKET
118 get_packet_from_freelist(struct xennet_info *xi)
119 {
120 NDIS_STATUS status;
121 PNDIS_PACKET packet;
123 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
125 if (!xi->rx_packet_free)
126 {
127 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
128 if (status != NDIS_STATUS_SUCCESS)
129 {
130 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
131 return NULL;
132 }
133 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
134 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
135 }
136 else
137 {
138 xi->rx_packet_free--;
139 packet = xi->rx_packet_list[xi->rx_packet_free];
140 }
141 return packet;
142 }
144 static VOID
145 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
146 {
147 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
148 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
150 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
151 {
152 KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
153 NdisFreePacket(packet);
154 return;
155 }
156 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
157 packet, TcpIpChecksumPacketInfo);
158 csum_info->Value = 0;
159 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
160 xi->rx_packet_list[xi->rx_packet_free] = packet;
161 xi->rx_packet_free++;
162 }
164 static VOID
165 packet_freelist_dispose(struct xennet_info *xi)
166 {
167 while(xi->rx_packet_free != 0)
168 {
169 xi->rx_packet_free--;
170 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
171 }
172 }
174 static PNDIS_PACKET
175 XenNet_MakePacket(struct xennet_info *xi)
176 {
177 NDIS_STATUS status;
178 PNDIS_PACKET packet;
179 PNDIS_BUFFER out_buffer;
180 USHORT new_ip4_length;
181 PUCHAR header_va;
182 packet_info_t *pi = &xi->rxpi;
184 //FUNCTION_ENTER();
186 if (!pi->split_required)
187 {
188 PNDIS_BUFFER buffer;
189 shared_buffer_t *page_buf;
191 //KdPrint((__DRIVER_NAME " !split_required\n"));
193 packet = get_packet_from_freelist(xi);
194 if (packet == NULL)
195 {
196 /* buffers will be freed in MakePackets */
197 KdPrint((__DRIVER_NAME " No free packets\n"));
198 //FUNCTION_EXIT();
199 return NULL;
200 }
201 xi->rx_outstanding++;
203 // what if we needed to consolidate the header? maybe should put that on instead of all the buffers...
204 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = pi->first_pb;
206 buffer = pi->first_buffer;
207 page_buf = pi->first_pb;
208 //KdPrint((__DRIVER_NAME " packet = %p, first_buffer = %p, first_pb = %p\n", packet, buffer, page_buf));
209 while (buffer)
210 {
211 PNDIS_BUFFER next_buffer;
212 //KdPrint((__DRIVER_NAME " buffer = %p\n", buffer));
214 NdisGetNextBuffer(buffer, &next_buffer);
215 NDIS_BUFFER_LINKAGE(buffer) = NULL;
216 NdisChainBufferAtBack(packet, buffer);
217 //ref_pb(xi, page_buf);
219 buffer = next_buffer;
220 page_buf = page_buf->next;
221 }
223 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
224 }
225 else
226 {
227 ULONG out_remaining;
228 shared_buffer_t *header_buf;
230 //KdPrint((__DRIVER_NAME " split_required\n"));
232 packet = get_packet_from_freelist(xi);
233 if (packet == NULL)
234 {
235 //FUNCTION_EXIT();
236 return NULL;
237 }
238 xi->rx_outstanding++;
240 //status = NdisAllocateMemoryWithTag((PUCHAR *)&header_buf, sizeof(shared_buffer_t) + pi->header_length, XENNET_POOL_TAG);
241 ASSERT(sizeof(shared_buffer_t) + pi->header_length < LOOKASIDE_LIST_ALLOC_SIZE);
242 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
243 ASSERT(header_buf); // lazy
244 header_va = (PUCHAR)(header_buf + 1);
245 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
246 NdisMoveMemory(header_va, pi->header, pi->header_length);
248 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too
250 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
251 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
252 NdisChainBufferAtBack(packet, out_buffer);
253 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = header_buf;
254 header_buf->next = pi->curr_pb;
256 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
257 //KdPrint((__DRIVER_NAME " curr_mdl_offset = %d\n", pi->curr_mdl_offset));
258 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
260 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
261 out_remaining = (USHORT)min(pi->mss, pi->tcp_remaining);
262 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + out_remaining);
263 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
264 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
265 pi->tcp_seq += out_remaining;
266 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - out_remaining);
267 do
268 {
269 ULONG in_buffer_offset;
270 ULONG in_buffer_length;
271 ULONG out_length;
272 //UINT tmp_packet_length;
273 //KdPrint((__DRIVER_NAME " curr_buffer = %p\n", pi->curr_buffer));
274 //KdPrint((__DRIVER_NAME " curr_pb = %p\n", pi->curr_pb));
275 //KdPrint((__DRIVER_NAME " out_remaining = %d\n", out_remaining));
276 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
277 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
278 //KdPrint((__DRIVER_NAME " out_length = %d\n", out_length));
279 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
280 //TODO: check status
281 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
282 NdisChainBufferAtBack(packet, out_buffer);
283 //NdisQueryPacketLength(packet, &tmp_packet_length);
284 //KdPrint((__DRIVER_NAME " current packet length = %d\n", tmp_packet_length));
285 ref_pb(xi, pi->curr_pb);
286 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
287 if (pi->curr_mdl_offset == in_buffer_length)
288 {
289 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
290 pi->curr_pb = pi->curr_pb->next;
291 pi->curr_mdl_offset = 0;
292 }
293 out_remaining -= out_length;
294 } while (out_remaining != 0);
295 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
296 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
297 }
299 //FUNCTION_EXIT();
300 return packet;
301 }
303 /*
304 Windows appears to insist that the checksum on received packets is correct, and won't
305 believe us when we lie about it, which happens when the packet is generated on the
306 same bridge in Dom0. Doh!
307 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
308 */
310 static BOOLEAN
311 XenNet_SumPacketData(
312 packet_info_t *pi,
313 PNDIS_PACKET packet,
314 BOOLEAN set_csum
315 )
316 {
317 USHORT i;
318 PUCHAR buffer;
319 PMDL mdl;
320 UINT total_length;
321 UINT buffer_length;
322 USHORT buffer_offset;
323 ULONG csum;
324 PUSHORT csum_ptr;
325 USHORT remaining;
326 USHORT ip4_length;
328 //FUNCTION_ENTER();
330 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
331 ASSERT(mdl);
333 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
335 if ((USHORT)(ip4_length + XN_HDR_SIZE) != total_length)
336 {
337 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
338 }
340 switch (pi->ip_proto)
341 {
342 case 6:
343 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
344 break;
345 case 17:
346 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
347 break;
348 default:
349 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
350 //FUNCTION_EXIT();
351 return FALSE; // should never happen
352 }
354 if (set_csum)
355 *csum_ptr = 0;
357 csum = 0;
358 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
359 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
360 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
362 remaining = ip4_length - pi->ip4_header_length;
364 csum += remaining;
366 for (buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length; i < total_length - 1; i += 2, buffer_offset += 2)
367 {
368 /* don't include the checksum field itself in the calculation */
369 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
370 continue;
371 if (buffer_offset == buffer_length - 1) // deal with a buffer ending on an odd byte boundary
372 {
373 csum += (USHORT)buffer[buffer_offset] << 8;
374 NdisGetNextBuffer(mdl, &mdl);
375 if (mdl == NULL)
376 {
377 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
378 return FALSE; // should never happen
379 }
380 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
381 csum += ((USHORT)buffer[0]);
382 buffer_offset = (USHORT)-1;
383 }
384 else
385 {
386 if (buffer_offset == buffer_length)
387 {
388 // KdPrint((__DRIVER_NAME " New buffer - aligned...\n"));
389 NdisGetNextBuffer(mdl, &mdl);
390 if (mdl == NULL)
391 {
392 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
393 return FALSE; // should never happen
394 }
395 NdisQueryBufferSafe(mdl, (PVOID) &buffer, &buffer_length, NormalPagePriority);
396 buffer_offset = 0;
397 }
398 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
399 }
400 }
401 if (i != total_length) // last odd byte
402 {
403 if (buffer_offset >= buffer_length)
404 {
405 NdisGetNextBuffer(mdl, &mdl);
406 NdisQueryBufferSafe(mdl, (PVOID)&buffer, &buffer_length, NormalPagePriority);
407 buffer_offset = 0;
408 }
409 csum += ((USHORT)buffer[buffer_offset] << 8);
410 }
411 while (csum & 0xFFFF0000)
412 csum = (csum & 0xFFFF) + (csum >> 16);
414 if (set_csum)
415 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
416 else
417 {
418 //FUNCTION_EXIT();
419 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
420 }
421 //FUNCTION_EXIT();
422 return TRUE;
423 }
425 static ULONG
426 XenNet_MakePackets(
427 struct xennet_info *xi,
428 PLIST_ENTRY rx_packet_list
429 )
430 {
431 USHORT i;
432 ULONG packet_count = 0;
433 PNDIS_PACKET packet;
434 PLIST_ENTRY entry;
435 UCHAR psh;
436 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
437 ULONG parse_result;
438 packet_info_t *pi = &xi->rxpi;
439 PNDIS_BUFFER buffer;
440 shared_buffer_t *page_buf;
442 //FUNCTION_ENTER();
444 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
446 if ((xi->packet_filter & NDIS_PACKET_TYPE_MULTICAST)
447 && !(xi->packet_filter & NDIS_PACKET_TYPE_ALL_MULTICAST)
448 && (pi->header[0] & 0x01)
449 && !(pi->header[0] == 0xFF && pi->header[1] == 0xFF && pi->header[2] == 0xFF
450 && pi->header[3] == 0xFF && pi->header[4] == 0xFF && pi->header[5] == 0xFF))
451 {
452 for (i = 0; i < xi->multicast_list_size; i++)
453 {
454 if (memcmp(xi->multicast_list[i], pi->header, 6) == 0)
455 break;
456 }
457 if (i == xi->multicast_list_size)
458 {
459 //KdPrint((__DRIVER_NAME " Packet not for my MAC address\n"));
460 goto done;
461 }
462 }
463 switch (pi->ip_proto)
464 {
465 case 6: // TCP
466 if (pi->split_required)
467 break;
468 // fallthrough
469 case 17: // UDP
470 packet = XenNet_MakePacket(xi);
471 if (packet == NULL)
472 {
473 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
474 xi->stat_rx_no_buffer++;
475 packet_count = 0;
476 goto done;
477 }
479 if (parse_result == PARSE_OK)
480 {
481 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
482 packet, TcpIpChecksumPacketInfo);
483 if (pi->csum_blank || pi->data_validated)
484 {
485 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
486 {
487 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
488 {
489 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
490 }
491 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
492 {
493 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
494 }
495 if (pi->csum_blank)
496 {
497 XenNet_SumPacketData(pi, packet, TRUE);
498 }
499 }
500 else if (xi->config_csum_rx_check)
501 {
502 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
503 {
504 if (XenNet_SumPacketData(pi, packet, FALSE))
505 {
506 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
507 }
508 else
509 {
510 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
511 }
512 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
513 {
514 if (XenNet_SumPacketData(pi, packet, FALSE))
515 {
516 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
517 }
518 else
519 {
520 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
521 }
522 }
523 }
524 }
525 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
526 InsertTailList(rx_packet_list, entry);
527 XenNet_ClearPacketInfo(pi);
528 //FUNCTION_EXIT();
529 return 1;
530 default:
531 packet = XenNet_MakePacket(xi);
532 if (packet == NULL)
533 {
534 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
535 xi->stat_rx_no_buffer++;
536 packet_count = 0;
537 goto done;
538 }
539 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
540 InsertTailList(rx_packet_list, entry);
541 XenNet_ClearPacketInfo(pi);
542 //FUNCTION_EXIT();
543 return 1;
544 }
546 /* synchronise the pb with the buffer */
547 buffer = pi->first_buffer;
548 pi->curr_pb = pi->first_pb;
549 //KdPrint((__DRIVER_NAME " first_buffer = %p, curr_buffer = %p, first_pb = %p, curr_pb = %p\n",
550 // pi->first_buffer, pi->curr_buffer, pi->first_pb, pi->curr_pb));
552 while (pi->curr_pb != NULL && buffer != pi->curr_buffer)
553 {
554 NdisGetNextBuffer(buffer, &buffer);
555 pi->curr_pb = pi->curr_pb->next;
556 //KdPrint((__DRIVER_NAME " buffer = %p, curr_pb = %p\n", buffer, pi->curr_pb));
557 }
559 pi->tcp_remaining = pi->tcp_length;
561 /* we can make certain assumptions here as the following code is only for tcp4 */
562 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
563 while (pi->tcp_remaining)
564 {
565 PUCHAR header_va;
566 PMDL mdl;
567 UINT total_length;
568 UINT buffer_length;
569 packet = XenNet_MakePacket(xi);
570 if (!packet)
571 {
572 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
573 xi->stat_rx_no_buffer++;
574 break; /* we are out of memory - just drop the packets */
575 }
576 if (xi->setting_csum.V4Receive.TcpChecksum)
577 {
578 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
579 packet, TcpIpChecksumPacketInfo);
580 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
581 }
582 if (psh)
583 {
584 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
585 if (pi->tcp_remaining)
586 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
587 else
588 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
589 }
590 XenNet_SumPacketData(pi, packet, TRUE);
591 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
592 InsertTailList(rx_packet_list, entry);
593 packet_count++;
594 }
596 done:
597 //buffer = pi->first_buffer;
598 page_buf = pi->first_pb;
599 while (page_buf)
600 {
601 //PNDIS_BUFFER next_buffer;
602 shared_buffer_t *next_pb;
604 //NdisGetNextBuffer(buffer, &next_buffer);
605 next_pb = page_buf->next;
607 //NdisFreeBuffer(buffer);
608 put_pb_on_freelist(xi, page_buf);
610 //buffer = next_buffer;
611 page_buf = next_pb;
612 }
613 XenNet_ClearPacketInfo(pi);
614 //FUNCTION_EXIT();
615 return packet_count;
616 }
618 static BOOLEAN
619 XenNet_RxQueueDpcSynchronized(PVOID context)
620 {
621 struct xennet_info *xi = context;
623 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
625 return TRUE;
626 }
628 #define MAXIMUM_PACKETS_PER_INDICATE 32
629 /*
630 We limit the number of packets per interrupt so that acks get a chance
631 under high rx load. The DPC is immediately re-scheduled */
633 #define MAX_PACKETS_PER_INTERRUPT 64
635 // Called at DISPATCH_LEVEL
636 static VOID
637 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
638 {
639 struct xennet_info *xi = context;
640 RING_IDX cons, prod;
641 LIST_ENTRY rx_packet_list;
642 PLIST_ENTRY entry;
643 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
644 ULONG packet_count = 0;
645 struct netif_rx_response *rxrsp = NULL;
646 struct netif_extra_info *ei;
647 USHORT id;
648 int more_to_do = FALSE;
649 packet_info_t *pi = &xi->rxpi;
650 //NDIS_STATUS status;
651 shared_buffer_t *page_buf;
652 PNDIS_BUFFER buffer;
654 UNREFERENCED_PARAMETER(dpc);
655 UNREFERENCED_PARAMETER(arg1);
656 UNREFERENCED_PARAMETER(arg2);
658 //FUNCTION_ENTER();
660 ASSERT(xi->connected);
662 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
664 if (xi->rx_shutting_down)
665 {
666 /* there is a chance that our Dpc had been queued just before the shutdown... */
667 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
668 return;
669 }
670 InitializeListHead(&rx_packet_list);
672 do {
673 prod = xi->rx.sring->rsp_prod;
674 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
675 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
677 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
678 {
679 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
680 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
681 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
682 xi->rx_ring_pbs[id] = 0xFFFF;
683 xi->rx_id_free++;
684 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
685 if (pi->extra_info)
686 {
687 //KdPrint((__DRIVER_NAME " processing extra info\n"));
688 put_pb_on_freelist(xi, page_buf);
689 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
690 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
691 switch (ei->type)
692 {
693 case XEN_NETIF_EXTRA_TYPE_GSO:
694 switch (ei->u.gso.type)
695 {
696 case XEN_NETIF_GSO_TYPE_TCPV4:
697 pi->mss = ei->u.gso.size;
698 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
699 break;
700 default:
701 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
702 break;
703 }
704 break;
705 default:
706 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
707 break;
708 }
709 }
710 else
711 {
712 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
713 //KdPrint((__DRIVER_NAME " status = %d, offset = %d\n", rxrsp->status, rxrsp->offset));
714 if (rxrsp->status <= 0
715 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
716 {
717 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
718 rxrsp->offset, rxrsp->status));
719 ASSERT(!pi->extra_info);
720 put_pb_on_freelist(xi, page_buf);
721 continue;
722 }
723 ASSERT(!rxrsp->offset);
724 ASSERT(rxrsp->id == id);
725 if (!pi->more_frags) // handling the packet's 1st buffer
726 {
727 if (rxrsp->flags & NETRXF_csum_blank)
728 pi->csum_blank = TRUE;
729 if (rxrsp->flags & NETRXF_data_validated)
730 pi->data_validated = TRUE;
731 }
732 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
733 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
734 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
735 buffer = page_buf->buffer;
736 NdisAdjustBufferLength(buffer, rxrsp->status);
737 if (pi->first_pb)
738 {
739 //KdPrint((__DRIVER_NAME " additional buffer\n"));
740 pi->curr_pb->next = page_buf;
741 pi->curr_pb = page_buf;
742 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
743 pi->curr_buffer = buffer;
744 }
745 else
746 {
747 pi->first_pb = page_buf;
748 pi->curr_pb = page_buf;
749 pi->first_buffer = buffer;
750 pi->curr_buffer = buffer;
751 }
752 pi->mdl_count++;
753 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
754 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
755 pi->total_length = pi->total_length + rxrsp->status;
756 }
758 /* Packet done, add it to the list */
759 if (!pi->more_frags && !pi->extra_info)
760 {
761 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
762 }
763 }
764 xi->rx.rsp_cons = cons;
766 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
767 break;
769 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
770 if (!more_to_do)
771 {
772 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
773 mb();
774 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
775 }
776 } while (more_to_do);
778 if (pi->more_frags || pi->extra_info)
779 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
781 /* Give netback more buffers */
782 XenNet_FillRing(xi);
784 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
785 {
786 /* fire again immediately */
787 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
788 }
790 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
792 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
794 entry = RemoveHeadList(&rx_packet_list);
795 packet_count = 0;
796 while (entry != &rx_packet_list)
797 {
798 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[0]);
799 packets[packet_count++] = packet;
800 entry = RemoveHeadList(&rx_packet_list);
801 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
802 {
803 //KdPrint((__DRIVER_NAME " Indicating\n"));
804 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
805 packet_count = 0;
806 }
807 }
808 //FUNCTION_EXIT();
809 }
811 /* called at DISPATCH_LEVEL */
812 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
813 VOID DDKAPI
814 XenNet_ReturnPacket(
815 IN NDIS_HANDLE MiniportAdapterContext,
816 IN PNDIS_PACKET Packet
817 )
818 {
819 struct xennet_info *xi = MiniportAdapterContext;
820 PNDIS_BUFFER buffer;
821 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[sizeof(LIST_ENTRY)];
823 //FUNCTION_ENTER();
825 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
827 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
829 NdisUnchainBufferAtFront(Packet, &buffer);
830 while (buffer)
831 {
832 shared_buffer_t *next_buf = page_buf->next;
833 if (!page_buf->virtual)
834 {
835 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
836 PUCHAR va;
837 UINT len;
838 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
839 //KdPrint((__DRIVER_NAME " freeing header buffer %p\n", va - sizeof(shared_buffer_t)));
840 //NdisFreeMemory(va - sizeof(shared_buffer_t), len + sizeof(shared_buffer_t), 0);
841 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
842 NdisFreeBuffer(buffer);
843 }
844 else
845 {
846 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
847 if (buffer != page_buf->buffer)
848 NdisFreeBuffer(buffer);
849 put_pb_on_freelist(xi, page_buf);
850 }
851 NdisUnchainBufferAtFront(Packet, &buffer);
852 page_buf = next_buf;
853 }
855 put_packet_on_freelist(xi, Packet);
856 xi->rx_outstanding--;
858 if (!xi->rx_outstanding && xi->rx_shutting_down)
859 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
861 XenNet_FillRing(xi);
863 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
865 //FUNCTION_EXIT();
866 }
868 /*
869 Free all Rx buffers (on halt, for example)
870 The ring must be stopped at this point.
871 */
873 static VOID
874 XenNet_PurgeRing(struct xennet_info *xi)
875 {
876 int i;
877 for (i = 0; i < NET_RX_RING_SIZE; i++)
878 {
879 if (xi->rx_ring_pbs[i] != 0xFFFF)
880 {
881 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
882 xi->rx_ring_pbs[i] = 0xFFFF;
883 }
884 }
885 }
887 static VOID
888 XenNet_BufferFree(struct xennet_info *xi)
889 {
890 shared_buffer_t *pb;
892 XenNet_PurgeRing(xi);
894 while ((pb = get_pb_from_freelist(xi)) != NULL)
895 {
896 NdisFreeBuffer(pb->buffer);
897 NdisMFreeSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, pb->virtual, pb->logical);
898 }
899 }
901 VOID
902 XenNet_RxResumeStart(xennet_info_t *xi)
903 {
904 KIRQL old_irql;
906 FUNCTION_ENTER();
908 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
909 XenNet_PurgeRing(xi);
910 KeReleaseSpinLock(&xi->rx_lock, old_irql);
912 FUNCTION_EXIT();
913 }
915 VOID
916 XenNet_BufferAlloc(xennet_info_t *xi)
917 {
918 NDIS_STATUS status;
919 int i;
921 xi->rx_id_free = NET_RX_RING_SIZE;
922 xi->rx_outstanding = 0;
924 for (i = 0; i < NET_RX_RING_SIZE; i++)
925 {
926 xi->rx_ring_pbs[i] = 0xFFFF;
927 }
929 for (i = 0; i < RX_PAGE_BUFFERS; i++)
930 {
931 xi->rx_pbs[i].id = (USHORT)i;
932 NdisMAllocateSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, &xi->rx_pbs[i].virtual, &xi->rx_pbs[i].logical);
933 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
934 if (status != STATUS_SUCCESS)
935 break;
936 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
937 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
938 }
939 if (i == 0)
940 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
941 }
944 VOID
945 XenNet_RxResumeEnd(xennet_info_t *xi)
946 {
947 KIRQL old_irql;
949 FUNCTION_ENTER();
951 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
952 //XenNet_BufferAlloc(xi);
953 XenNet_FillRing(xi);
954 KeReleaseSpinLock(&xi->rx_lock, old_irql);
956 FUNCTION_EXIT();
957 }
959 BOOLEAN
960 XenNet_RxInit(xennet_info_t *xi)
961 {
962 NDIS_STATUS status;
964 FUNCTION_ENTER();
966 xi->rx_shutting_down = FALSE;
967 KeInitializeSpinLock(&xi->rx_lock);
968 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
969 KeInitializeTimer(&xi->rx_timer);
970 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
971 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
972 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
973 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
975 XenNet_BufferAlloc(xi);
977 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
978 PROTOCOL_RESERVED_SIZE_IN_PACKET);
979 if (status != NDIS_STATUS_SUCCESS)
980 {
981 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
982 return FALSE;
983 }
985 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0, LOOKASIDE_LIST_ALLOC_SIZE, XENNET_POOL_TAG, 0);
987 XenNet_FillRing(xi);
989 FUNCTION_EXIT();
991 return TRUE;
992 }
994 BOOLEAN
995 XenNet_RxShutdown(xennet_info_t *xi)
996 {
997 KIRQL old_irql;
999 FUNCTION_ENTER();
1001 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1002 xi->rx_shutting_down = TRUE;
1003 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1005 if (xi->config_rx_interrupt_moderation)
1007 KeCancelTimer(&xi->rx_timer);
1010 KeRemoveQueueDpc(&xi->rx_dpc);
1011 KeFlushQueuedDpcs();
1013 while (xi->rx_outstanding)
1015 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1016 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1019 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1021 XenNet_BufferFree(xi);
1023 packet_freelist_dispose(xi);
1025 NdisFreePacketPool(xi->rx_packet_pool);
1027 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1029 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1031 FUNCTION_EXIT();
1033 return TRUE;