win-pvdrivers

view xennet/xennet_rx.c @ 584:40a40d4aa88a

Fix a possible Dpc race in xennet_rx.
Remove some old code.
author James Harper <james.harper@bendigoit.com.au>
date Sun Jun 07 20:19:20 2009 +1000 (2009-06-07)
parents dc0a293c870c
children bdb2d6b163f3
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 static __inline shared_buffer_t *
24 get_pb_from_freelist(struct xennet_info *xi)
25 {
26 shared_buffer_t *pb;
28 if (xi->rx_pb_free == 0)
29 {
30 KdPrint((__DRIVER_NAME " Out of pb's\n"));
31 return NULL;
32 }
33 xi->rx_pb_free--;
35 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
36 pb->ref_count++;
37 return pb;
38 }
40 static __inline VOID
41 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
42 {
43 UNREFERENCED_PARAMETER(xi);
44 pb->ref_count++;
45 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
46 }
48 static __inline VOID
49 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
50 {
51 pb->ref_count--;
52 //KdPrint((__DRIVER_NAME " decremented pb %p ref to %d\n", pb, pb->ref_count));
53 if (pb->ref_count == 0)
54 {
55 //KdPrint((__DRIVER_NAME " freeing pb %p\n", pb, pb->ref_count));
56 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
57 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
58 pb->next = NULL;
59 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
60 xi->rx_pb_free++;
61 }
62 }
64 // Called at DISPATCH_LEVEL with rx lock held
65 static NDIS_STATUS
66 XenNet_FillRing(struct xennet_info *xi)
67 {
68 unsigned short id;
69 shared_buffer_t *page_buf;
70 ULONG i, notify;
71 ULONG batch_target;
72 RING_IDX req_prod = xi->rx.req_prod_pvt;
73 netif_rx_request_t *req;
75 //FUNCTION_ENTER();
77 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
79 if (batch_target < (xi->rx_target >> 2))
80 {
81 //FUNCTION_EXIT();
82 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
83 }
85 for (i = 0; i < batch_target; i++)
86 {
87 page_buf = get_pb_from_freelist(xi);
88 if (!page_buf)
89 {
90 KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
91 break;
92 }
93 xi->rx_id_free--;
95 /* Give to netback */
96 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
97 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
98 xi->rx_ring_pbs[id] = page_buf->id;
99 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
100 req->id = id;
101 req->gref = (grant_ref_t)(page_buf->logical.QuadPart >> PAGE_SHIFT);
102 ASSERT(req->gref != INVALID_GRANT_REF);
103 }
104 KeMemoryBarrier();
105 xi->rx.req_prod_pvt = req_prod + i;
106 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
107 if (notify)
108 {
109 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
110 }
112 //FUNCTION_EXIT();
114 return NDIS_STATUS_SUCCESS;
115 }
117 static PNDIS_PACKET
118 get_packet_from_freelist(struct xennet_info *xi)
119 {
120 NDIS_STATUS status;
121 PNDIS_PACKET packet;
123 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
125 if (!xi->rx_packet_free)
126 {
127 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
128 if (status != NDIS_STATUS_SUCCESS)
129 {
130 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
131 return NULL;
132 }
133 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
134 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
135 }
136 else
137 {
138 xi->rx_packet_free--;
139 packet = xi->rx_packet_list[xi->rx_packet_free];
140 }
141 return packet;
142 }
144 static VOID
145 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
146 {
147 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
148 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
150 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
151 {
152 KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
153 NdisFreePacket(packet);
154 return;
155 }
156 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
157 packet, TcpIpChecksumPacketInfo);
158 csum_info->Value = 0;
159 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
160 xi->rx_packet_list[xi->rx_packet_free] = packet;
161 xi->rx_packet_free++;
162 }
164 static VOID
165 packet_freelist_dispose(struct xennet_info *xi)
166 {
167 while(xi->rx_packet_free != 0)
168 {
169 xi->rx_packet_free--;
170 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
171 }
172 }
174 static PNDIS_PACKET
175 XenNet_MakePacket(struct xennet_info *xi)
176 {
177 NDIS_STATUS status;
178 PNDIS_PACKET packet;
179 PNDIS_BUFFER out_buffer;
180 USHORT new_ip4_length;
181 PUCHAR header_va;
182 packet_info_t *pi = &xi->rxpi;
184 //FUNCTION_ENTER();
186 if (!pi->split_required)
187 {
188 PNDIS_BUFFER buffer;
189 shared_buffer_t *page_buf;
191 //KdPrint((__DRIVER_NAME " !split_required\n"));
193 packet = get_packet_from_freelist(xi);
194 if (packet == NULL)
195 {
196 /* buffers will be freed in MakePackets */
197 KdPrint((__DRIVER_NAME " No free packets\n"));
198 //FUNCTION_EXIT();
199 return NULL;
200 }
201 xi->rx_outstanding++;
203 // what if we needed to consolidate the header? maybe should put that on instead of all the buffers...
204 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = pi->first_pb;
206 buffer = pi->first_buffer;
207 page_buf = pi->first_pb;
208 //KdPrint((__DRIVER_NAME " packet = %p, first_buffer = %p, first_pb = %p\n", packet, buffer, page_buf));
209 while (buffer)
210 {
211 PNDIS_BUFFER next_buffer;
212 //KdPrint((__DRIVER_NAME " buffer = %p\n", buffer));
214 NdisGetNextBuffer(buffer, &next_buffer);
215 NDIS_BUFFER_LINKAGE(buffer) = NULL;
216 NdisChainBufferAtBack(packet, buffer);
217 //ref_pb(xi, page_buf);
219 buffer = next_buffer;
220 page_buf = page_buf->next;
221 }
223 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
224 }
225 else
226 {
227 ULONG out_remaining;
228 shared_buffer_t *header_buf;
230 //KdPrint((__DRIVER_NAME " split_required\n"));
232 packet = get_packet_from_freelist(xi);
233 if (packet == NULL)
234 {
235 //FUNCTION_EXIT();
236 return NULL;
237 }
238 xi->rx_outstanding++;
240 //status = NdisAllocateMemoryWithTag((PUCHAR *)&header_buf, sizeof(shared_buffer_t) + pi->header_length, XENNET_POOL_TAG);
241 ASSERT(sizeof(shared_buffer_t) + pi->header_length < LOOKASIDE_LIST_ALLOC_SIZE);
242 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
243 ASSERT(header_buf); // lazy
244 header_va = (PUCHAR)(header_buf + 1);
245 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
246 NdisMoveMemory(header_va, pi->header, pi->header_length);
248 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too
250 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
251 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
252 NdisChainBufferAtBack(packet, out_buffer);
253 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = header_buf;
254 header_buf->next = pi->curr_pb;
256 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
257 //KdPrint((__DRIVER_NAME " curr_mdl_offset = %d\n", pi->curr_mdl_offset));
258 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
260 //KdPrint((__DRIVER_NAME " tcp_remaining = %d\n", pi->tcp_remaining));
261 out_remaining = (USHORT)min(pi->mss, pi->tcp_remaining);
262 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + out_remaining);
263 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
264 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
265 pi->tcp_seq += out_remaining;
266 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - out_remaining);
267 do
268 {
269 ULONG in_buffer_offset;
270 ULONG in_buffer_length;
271 ULONG out_length;
272 //UINT tmp_packet_length;
273 //KdPrint((__DRIVER_NAME " curr_buffer = %p\n", pi->curr_buffer));
274 //KdPrint((__DRIVER_NAME " curr_pb = %p\n", pi->curr_pb));
275 //KdPrint((__DRIVER_NAME " out_remaining = %d\n", out_remaining));
276 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
277 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
278 //KdPrint((__DRIVER_NAME " out_length = %d\n", out_length));
279 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
280 //TODO: check status
281 //KdPrint((__DRIVER_NAME " about to add buffer with length = %d\n", MmGetMdlByteCount(out_buffer)));
282 NdisChainBufferAtBack(packet, out_buffer);
283 //NdisQueryPacketLength(packet, &tmp_packet_length);
284 //KdPrint((__DRIVER_NAME " current packet length = %d\n", tmp_packet_length));
285 ref_pb(xi, pi->curr_pb);
286 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
287 if (pi->curr_mdl_offset == in_buffer_length)
288 {
289 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
290 pi->curr_pb = pi->curr_pb->next;
291 pi->curr_mdl_offset = 0;
292 }
293 out_remaining -= out_length;
294 } while (out_remaining != 0);
295 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
296 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
297 }
299 //FUNCTION_EXIT();
300 return packet;
301 }
303 /*
304 Windows appears to insist that the checksum on received packets is correct, and won't
305 believe us when we lie about it, which happens when the packet is generated on the
306 same bridge in Dom0. Doh!
307 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
308 */
310 static BOOLEAN
311 XenNet_SumPacketData(
312 packet_info_t *pi,
313 PNDIS_PACKET packet,
314 BOOLEAN set_csum
315 )
316 {
317 USHORT i;
318 PUCHAR buffer;
319 PMDL mdl;
320 UINT total_length;
321 UINT buffer_length;
322 USHORT buffer_offset;
323 ULONG csum;
324 PUSHORT csum_ptr;
325 USHORT remaining;
326 USHORT ip4_length;
328 //FUNCTION_ENTER();
330 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
331 ASSERT(mdl);
333 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
335 if ((USHORT)(ip4_length + XN_HDR_SIZE) != total_length)
336 {
337 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
338 }
340 switch (pi->ip_proto)
341 {
342 case 6:
343 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
344 break;
345 case 17:
346 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
347 break;
348 default:
349 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
350 //FUNCTION_EXIT();
351 return FALSE; // should never happen
352 }
354 if (set_csum)
355 *csum_ptr = 0;
357 csum = 0;
358 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
359 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
360 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
362 remaining = ip4_length - pi->ip4_header_length;
364 csum += remaining;
366 for (buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length; i < total_length - 1; i += 2, buffer_offset += 2)
367 {
368 /* don't include the checksum field itself in the calculation */
369 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
370 continue;
371 if (buffer_offset == buffer_length - 1) // deal with a buffer ending on an odd byte boundary
372 {
373 csum += (USHORT)buffer[buffer_offset] << 8;
374 NdisGetNextBuffer(mdl, &mdl);
375 if (mdl == NULL)
376 {
377 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
378 return FALSE; // should never happen
379 }
380 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
381 csum += ((USHORT)buffer[0]);
382 buffer_offset = (USHORT)-1;
383 }
384 else
385 {
386 if (buffer_offset == buffer_length)
387 {
388 // KdPrint((__DRIVER_NAME " New buffer - aligned...\n"));
389 NdisGetNextBuffer(mdl, &mdl);
390 if (mdl == NULL)
391 {
392 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
393 return FALSE; // should never happen
394 }
395 NdisQueryBufferSafe(mdl, (PVOID) &buffer, &buffer_length, NormalPagePriority);
396 buffer_offset = 0;
397 }
398 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
399 }
400 }
401 if (i != total_length) // last odd byte
402 {
403 csum += ((USHORT)buffer[buffer_offset] << 8);
404 }
405 while (csum & 0xFFFF0000)
406 csum = (csum & 0xFFFF) + (csum >> 16);
408 if (set_csum)
409 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
410 else
411 {
412 //FUNCTION_EXIT();
413 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
414 }
415 //FUNCTION_EXIT();
416 return TRUE;
417 }
419 static ULONG
420 XenNet_MakePackets(
421 struct xennet_info *xi,
422 PLIST_ENTRY rx_packet_list
423 )
424 {
425 USHORT i;
426 ULONG packet_count = 0;
427 PNDIS_PACKET packet;
428 PLIST_ENTRY entry;
429 UCHAR psh;
430 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
431 ULONG parse_result;
432 packet_info_t *pi = &xi->rxpi;
433 PNDIS_BUFFER buffer;
434 shared_buffer_t *page_buf;
436 //FUNCTION_ENTER();
438 parse_result = XenNet_ParsePacketHeader(pi);
440 if ((xi->packet_filter & NDIS_PACKET_TYPE_MULTICAST)
441 && !(xi->packet_filter & NDIS_PACKET_TYPE_ALL_MULTICAST)
442 && (pi->header[0] & 0x01)
443 && !(pi->header[0] == 0xFF && pi->header[1] == 0xFF && pi->header[2] == 0xFF
444 && pi->header[3] == 0xFF && pi->header[4] == 0xFF && pi->header[5] == 0xFF))
445 {
446 for (i = 0; i < xi->multicast_list_size; i++)
447 {
448 if (memcmp(xi->multicast_list[i], pi->header, 6) == 0)
449 break;
450 }
451 if (i == xi->multicast_list_size)
452 {
453 //KdPrint((__DRIVER_NAME " Packet not for my MAC address\n"));
454 goto done;
455 }
456 }
457 switch (pi->ip_proto)
458 {
459 case 6: // TCP
460 if (pi->split_required)
461 break;
462 // fallthrough
463 case 17: // UDP
464 packet = XenNet_MakePacket(xi);
465 if (packet == NULL)
466 {
467 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
468 xi->stat_rx_no_buffer++;
469 packet_count = 0;
470 goto done;
471 }
473 if (parse_result == PARSE_OK)
474 {
475 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
476 packet, TcpIpChecksumPacketInfo);
477 if (pi->csum_blank || pi->data_validated)
478 {
479 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
480 {
481 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
482 {
483 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
484 }
485 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
486 {
487 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
488 }
489 if (pi->csum_blank)
490 {
491 XenNet_SumPacketData(pi, packet, TRUE);
492 }
493 }
494 else if (xi->config_csum_rx_check)
495 {
496 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
497 {
498 if (XenNet_SumPacketData(pi, packet, FALSE))
499 {
500 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
501 }
502 else
503 {
504 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
505 }
506 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
507 {
508 if (XenNet_SumPacketData(pi, packet, FALSE))
509 {
510 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
511 }
512 else
513 {
514 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
515 }
516 }
517 }
518 }
519 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
520 InsertTailList(rx_packet_list, entry);
521 XenNet_ClearPacketInfo(pi);
522 //FUNCTION_EXIT();
523 return 1;
524 default:
525 packet = XenNet_MakePacket(xi);
526 if (packet == NULL)
527 {
528 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
529 xi->stat_rx_no_buffer++;
530 packet_count = 0;
531 goto done;
532 }
533 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
534 InsertTailList(rx_packet_list, entry);
535 XenNet_ClearPacketInfo(pi);
536 //FUNCTION_EXIT();
537 return 1;
538 }
540 /* synchronise the pb with the buffer */
541 buffer = pi->first_buffer;
542 pi->curr_pb = pi->first_pb;
543 //KdPrint((__DRIVER_NAME " first_buffer = %p, curr_buffer = %p, first_pb = %p, curr_pb = %p\n",
544 // pi->first_buffer, pi->curr_buffer, pi->first_pb, pi->curr_pb));
546 while (pi->curr_pb != NULL && buffer != pi->curr_buffer)
547 {
548 NdisGetNextBuffer(buffer, &buffer);
549 pi->curr_pb = pi->curr_pb->next;
550 //KdPrint((__DRIVER_NAME " buffer = %p, curr_pb = %p\n", buffer, pi->curr_pb));
551 }
553 pi->tcp_remaining = pi->tcp_length;
555 /* we can make certain assumptions here as the following code is only for tcp4 */
556 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
557 while (pi->tcp_remaining)
558 {
559 PUCHAR header_va;
560 PMDL mdl;
561 UINT total_length;
562 UINT buffer_length;
563 packet = XenNet_MakePacket(xi);
564 if (!packet)
565 {
566 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
567 xi->stat_rx_no_buffer++;
568 break; /* we are out of memory - just drop the packets */
569 }
570 if (xi->setting_csum.V4Receive.TcpChecksum)
571 {
572 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
573 packet, TcpIpChecksumPacketInfo);
574 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
575 }
576 if (psh)
577 {
578 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
579 if (pi->tcp_remaining)
580 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
581 else
582 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
583 }
584 XenNet_SumPacketData(pi, packet, TRUE);
585 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
586 InsertTailList(rx_packet_list, entry);
587 packet_count++;
588 }
590 done:
591 //buffer = pi->first_buffer;
592 page_buf = pi->first_pb;
593 while (page_buf)
594 {
595 //PNDIS_BUFFER next_buffer;
596 shared_buffer_t *next_pb;
598 //NdisGetNextBuffer(buffer, &next_buffer);
599 next_pb = page_buf->next;
601 //NdisFreeBuffer(buffer);
602 put_pb_on_freelist(xi, page_buf);
604 //buffer = next_buffer;
605 page_buf = next_pb;
606 }
607 XenNet_ClearPacketInfo(pi);
608 //FUNCTION_EXIT();
609 return packet_count;
610 }
612 static BOOLEAN
613 XenNet_RxQueueDpcSynchronized(PVOID context)
614 {
615 struct xennet_info *xi = context;
617 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
619 return TRUE;
620 }
622 #define MAXIMUM_PACKETS_PER_INDICATE 32
623 /*
624 We limit the number of packets per interrupt so that acks get a chance
625 under high rx load. The DPC is immediately re-scheduled */
627 #define MAX_PACKETS_PER_INTERRUPT 64
629 // Called at DISPATCH_LEVEL
630 static VOID
631 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
632 {
633 struct xennet_info *xi = context;
634 RING_IDX cons, prod;
635 LIST_ENTRY rx_packet_list;
636 PLIST_ENTRY entry;
637 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
638 ULONG packet_count = 0;
639 struct netif_rx_response *rxrsp = NULL;
640 struct netif_extra_info *ei;
641 USHORT id;
642 int more_to_do = FALSE;
643 packet_info_t *pi = &xi->rxpi;
644 //NDIS_STATUS status;
645 shared_buffer_t *page_buf;
646 PNDIS_BUFFER buffer;
648 UNREFERENCED_PARAMETER(dpc);
649 UNREFERENCED_PARAMETER(arg1);
650 UNREFERENCED_PARAMETER(arg2);
652 //FUNCTION_ENTER();
654 ASSERT(xi->connected);
656 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
658 if (xi->shutting_down)
659 {
660 /* there is a chance that our Dpc had been queued just before the shutdown... */
661 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
662 return;
663 }
664 InitializeListHead(&rx_packet_list);
666 do {
667 prod = xi->rx.sring->rsp_prod;
668 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
669 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
671 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
672 {
673 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
674 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
675 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
676 xi->rx_ring_pbs[id] = 0xFFFF;
677 xi->rx_id_free++;
678 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
679 if (pi->extra_info)
680 {
681 //KdPrint((__DRIVER_NAME " processing extra info\n"));
682 put_pb_on_freelist(xi, page_buf);
683 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
684 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
685 switch (ei->type)
686 {
687 case XEN_NETIF_EXTRA_TYPE_GSO:
688 switch (ei->u.gso.type)
689 {
690 case XEN_NETIF_GSO_TYPE_TCPV4:
691 pi->mss = ei->u.gso.size;
692 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
693 break;
694 default:
695 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
696 break;
697 }
698 break;
699 default:
700 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
701 break;
702 }
703 }
704 else
705 {
706 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
707 //KdPrint((__DRIVER_NAME " status = %d, offset = %d\n", rxrsp->status, rxrsp->offset));
708 if (rxrsp->status <= 0
709 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
710 {
711 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
712 rxrsp->offset, rxrsp->status));
713 ASSERT(!pi->extra_info);
714 put_pb_on_freelist(xi, page_buf);
715 continue;
716 }
717 ASSERT(!rxrsp->offset);
718 ASSERT(rxrsp->id == id);
719 if (!pi->more_frags) // handling the packet's 1st buffer
720 {
721 if (rxrsp->flags & NETRXF_csum_blank)
722 pi->csum_blank = TRUE;
723 if (rxrsp->flags & NETRXF_data_validated)
724 pi->data_validated = TRUE;
725 }
726 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
727 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
728 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
729 buffer = page_buf->buffer;
730 NdisAdjustBufferLength(buffer, rxrsp->status);
731 if (pi->first_pb)
732 {
733 //KdPrint((__DRIVER_NAME " additional buffer\n"));
734 pi->curr_pb->next = page_buf;
735 pi->curr_pb = page_buf;
736 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
737 pi->curr_buffer = buffer;
738 }
739 else
740 {
741 pi->first_pb = page_buf;
742 pi->curr_pb = page_buf;
743 pi->first_buffer = buffer;
744 pi->curr_buffer = buffer;
745 }
746 pi->mdl_count++;
747 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
748 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
749 pi->total_length = pi->total_length + rxrsp->status;
750 }
752 /* Packet done, add it to the list */
753 if (!pi->more_frags && !pi->extra_info)
754 {
755 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
756 }
757 }
758 xi->rx.rsp_cons = cons;
760 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
761 break;
763 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
764 if (!more_to_do)
765 {
766 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
767 mb();
768 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
769 }
770 } while (more_to_do);
772 if (pi->more_frags || pi->extra_info)
773 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
775 /* Give netback more buffers */
776 XenNet_FillRing(xi);
778 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
779 {
780 /* fire again immediately */
781 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
782 }
784 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
786 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
788 entry = RemoveHeadList(&rx_packet_list);
789 packet_count = 0;
790 while (entry != &rx_packet_list)
791 {
792 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[0]);
793 packets[packet_count++] = packet;
794 entry = RemoveHeadList(&rx_packet_list);
795 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
796 {
797 //KdPrint((__DRIVER_NAME " Indicating\n"));
798 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
799 packet_count = 0;
800 }
801 }
802 //FUNCTION_EXIT();
803 }
805 /* called at DISPATCH_LEVEL */
806 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
807 VOID DDKAPI
808 XenNet_ReturnPacket(
809 IN NDIS_HANDLE MiniportAdapterContext,
810 IN PNDIS_PACKET Packet
811 )
812 {
813 struct xennet_info *xi = MiniportAdapterContext;
814 PNDIS_BUFFER buffer;
815 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[sizeof(LIST_ENTRY)];
817 //FUNCTION_ENTER();
819 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
821 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
823 NdisUnchainBufferAtFront(Packet, &buffer);
824 while (buffer)
825 {
826 shared_buffer_t *next_buf = page_buf->next;
827 if (!page_buf->virtual)
828 {
829 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
830 PUCHAR va;
831 UINT len;
832 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
833 //KdPrint((__DRIVER_NAME " freeing header buffer %p\n", va - sizeof(shared_buffer_t)));
834 //NdisFreeMemory(va - sizeof(shared_buffer_t), len + sizeof(shared_buffer_t), 0);
835 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
836 NdisFreeBuffer(buffer);
837 }
838 else
839 {
840 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
841 if (buffer != page_buf->buffer)
842 NdisFreeBuffer(buffer);
843 put_pb_on_freelist(xi, page_buf);
844 }
845 NdisUnchainBufferAtFront(Packet, &buffer);
846 page_buf = next_buf;
847 }
849 put_packet_on_freelist(xi, Packet);
850 xi->rx_outstanding--;
852 if (!xi->rx_outstanding && xi->shutting_down)
853 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
855 XenNet_FillRing(xi);
857 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
859 //FUNCTION_EXIT();
860 }
862 /*
863 Free all Rx buffers (on halt, for example)
864 The ring must be stopped at this point.
865 */
867 static VOID
868 XenNet_PurgeRing(struct xennet_info *xi)
869 {
870 int i;
871 for (i = 0; i < NET_RX_RING_SIZE; i++)
872 {
873 if (xi->rx_ring_pbs[i] != 0xFFFF)
874 {
875 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
876 xi->rx_ring_pbs[i] = 0xFFFF;
877 }
878 }
879 }
881 static VOID
882 XenNet_BufferFree(struct xennet_info *xi)
883 {
884 shared_buffer_t *pb;
886 XenNet_PurgeRing(xi);
888 while ((pb = get_pb_from_freelist(xi)) != NULL)
889 {
890 NdisFreeBuffer(pb->buffer);
891 NdisMFreeSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, pb->virtual, pb->logical);
892 }
893 }
895 VOID
896 XenNet_RxResumeStart(xennet_info_t *xi)
897 {
898 KIRQL old_irql;
900 FUNCTION_ENTER();
902 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
903 XenNet_PurgeRing(xi);
904 KeReleaseSpinLock(&xi->rx_lock, old_irql);
906 FUNCTION_EXIT();
907 }
909 VOID
910 XenNet_BufferAlloc(xennet_info_t *xi)
911 {
912 NDIS_STATUS status;
913 int i;
915 xi->rx_id_free = NET_RX_RING_SIZE;
916 xi->rx_outstanding = 0;
918 for (i = 0; i < NET_RX_RING_SIZE; i++)
919 {
920 xi->rx_ring_pbs[i] = 0xFFFF;
921 }
923 for (i = 0; i < RX_PAGE_BUFFERS; i++)
924 {
925 xi->rx_pbs[i].id = (USHORT)i;
926 NdisMAllocateSharedMemory(xi->adapter_handle, PAGE_SIZE, TRUE, &xi->rx_pbs[i].virtual, &xi->rx_pbs[i].logical);
927 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
928 if (status != STATUS_SUCCESS)
929 break;
930 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
931 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
932 }
933 if (i == 0)
934 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
935 }
938 VOID
939 XenNet_RxResumeEnd(xennet_info_t *xi)
940 {
941 KIRQL old_irql;
943 FUNCTION_ENTER();
945 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
946 //XenNet_BufferAlloc(xi);
947 XenNet_FillRing(xi);
948 KeReleaseSpinLock(&xi->rx_lock, old_irql);
950 FUNCTION_EXIT();
951 }
953 BOOLEAN
954 XenNet_RxInit(xennet_info_t *xi)
955 {
956 NDIS_STATUS status;
958 FUNCTION_ENTER();
960 KeInitializeSpinLock(&xi->rx_lock);
961 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
962 KeInitializeTimer(&xi->rx_timer);
963 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
964 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
965 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
966 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
968 XenNet_BufferAlloc(xi);
970 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
971 PROTOCOL_RESERVED_SIZE_IN_PACKET);
972 if (status != NDIS_STATUS_SUCCESS)
973 {
974 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
975 return FALSE;
976 }
978 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0, LOOKASIDE_LIST_ALLOC_SIZE, XENNET_POOL_TAG, 0);
980 XenNet_FillRing(xi);
982 FUNCTION_EXIT();
984 return TRUE;
985 }
987 BOOLEAN
988 XenNet_RxShutdown(xennet_info_t *xi)
989 {
990 KIRQL OldIrql;
992 FUNCTION_ENTER();
994 if (xi->config_rx_interrupt_moderation)
995 {
996 KeCancelTimer(&xi->rx_timer);
997 }
999 while (xi->rx_outstanding)
1001 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1002 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1005 KeAcquireSpinLock(&xi->rx_lock, &OldIrql);
1007 XenNet_BufferFree(xi);
1009 packet_freelist_dispose(xi);
1011 NdisFreePacketPool(xi->rx_packet_pool);
1013 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1015 KeReleaseSpinLock(&xi->rx_lock, OldIrql);
1017 FUNCTION_EXIT();
1019 return TRUE;