win-pvdrivers

view xennet/xennet_rx.c @ 790:467005e7f509

Big messy changes. Add grant ref tagging to better track when things go wrong (debug build only).
Fix a race in xennet that causes crashes under heavy traffic conditions on driver shutdown.
author James Harper <james.harper@bendigoit.com.au>
date Fri Mar 12 09:38:42 2010 +1100 (2010-03-12)
parents e0cfc7938e06
children 9c0c4210b778
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 /* Not really necessary but keeps PREfast happy */
24 static KDEFERRED_ROUTINE XenNet_RxBufferCheck;
26 static __inline shared_buffer_t *
27 get_pb_from_freelist(struct xennet_info *xi)
28 {
29 shared_buffer_t *pb;
31 if (xi->rx_pb_free == 0)
32 {
33 //KdPrint((__DRIVER_NAME " Out of pb's\n"));
34 return NULL;
35 }
36 xi->rx_pb_free--;
38 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
39 pb->ref_count++;
40 return pb;
41 }
43 static __inline VOID
44 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
45 {
46 UNREFERENCED_PARAMETER(xi);
47 pb->ref_count++;
48 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
49 }
51 static __inline VOID
52 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
53 {
54 pb->ref_count--;
55 if (pb->ref_count == 0)
56 {
57 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
58 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
59 pb->next = NULL;
60 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
61 xi->rx_pb_free++;
62 }
63 }
65 // Called at DISPATCH_LEVEL with rx lock held
66 static NDIS_STATUS
67 XenNet_FillRing(struct xennet_info *xi)
68 {
69 unsigned short id;
70 shared_buffer_t *page_buf;
71 ULONG i, notify;
72 ULONG batch_target;
73 RING_IDX req_prod = xi->rx.req_prod_pvt;
74 netif_rx_request_t *req;
76 //FUNCTION_ENTER();
78 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
80 if (batch_target < (xi->rx_target >> 2))
81 {
82 //FUNCTION_EXIT();
83 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
84 }
86 for (i = 0; i < batch_target; i++)
87 {
88 page_buf = get_pb_from_freelist(xi);
89 if (!page_buf)
90 {
91 //KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
92 break;
93 }
94 xi->rx_id_free--;
96 /* Give to netback */
97 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
98 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
99 xi->rx_ring_pbs[id] = page_buf->id;
100 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
101 req->id = id;
102 req->gref = page_buf->gref;
103 ASSERT(req->gref != INVALID_GRANT_REF);
104 }
105 KeMemoryBarrier();
106 xi->rx.req_prod_pvt = req_prod + i;
107 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
108 if (notify)
109 {
110 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
111 }
113 //FUNCTION_EXIT();
115 return NDIS_STATUS_SUCCESS;
116 }
118 static PNDIS_PACKET
119 get_packet_from_freelist(struct xennet_info *xi)
120 {
121 NDIS_STATUS status;
122 PNDIS_PACKET packet;
124 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
126 if (!xi->rx_packet_free)
127 {
128 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
129 if (status != NDIS_STATUS_SUCCESS)
130 {
131 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
132 return NULL;
133 }
134 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
135 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
136 }
137 else
138 {
139 xi->rx_packet_free--;
140 packet = xi->rx_packet_list[xi->rx_packet_free];
141 }
142 return packet;
143 }
145 static VOID
146 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
147 {
148 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
149 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
151 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
152 {
153 //KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
154 NdisFreePacket(packet);
155 return;
156 }
157 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
158 packet, TcpIpChecksumPacketInfo);
159 csum_info->Value = 0;
160 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
161 xi->rx_packet_list[xi->rx_packet_free] = packet;
162 xi->rx_packet_free++;
163 }
165 static VOID
166 packet_freelist_dispose(struct xennet_info *xi)
167 {
168 while(xi->rx_packet_free != 0)
169 {
170 xi->rx_packet_free--;
171 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
172 }
173 }
175 static PNDIS_PACKET
176 XenNet_MakePacket(struct xennet_info *xi)
177 {
178 NDIS_STATUS status;
179 PNDIS_PACKET packet;
180 PNDIS_BUFFER out_buffer;
181 USHORT new_ip4_length;
182 PUCHAR header_va;
183 packet_info_t *pi = &xi->rxpi;
184 ULONG out_remaining;
185 ULONG tcp_length;
186 ULONG header_extra;
187 shared_buffer_t *header_buf;
189 //FUNCTION_ENTER();
191 packet = get_packet_from_freelist(xi);
192 if (packet == NULL)
193 {
194 /* buffers will be freed in MakePackets */
195 KdPrint((__DRIVER_NAME " No free packets\n"));
196 //FUNCTION_EXIT();
197 return NULL;
198 }
200 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
201 ASSERT(header_buf); // lazy
202 header_va = (PUCHAR)(header_buf + 1);
203 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
204 NdisMoveMemory(header_va, pi->header, pi->header_length);
205 //KdPrint((__DRIVER_NAME " header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead));
206 //KdPrint((__DRIVER_NAME " ip4_header_length = %d\n", pi->ip4_header_length));
207 //KdPrint((__DRIVER_NAME " tcp_header_length = %d\n", pi->tcp_header_length));
208 /* make sure we satisfy the lookahead requirement */
209 XenNet_BuildHeader(pi, header_va, max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead) + MAX_ETH_HEADER_LENGTH);
210 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
211 ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
212 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
213 ASSERT(status == STATUS_SUCCESS);
214 NdisChainBufferAtBack(packet, out_buffer);
215 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = header_buf;
216 header_buf->next = pi->curr_pb;
219 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too... maybe
221 //KdPrint((__DRIVER_NAME " split_required = %d\n", pi->split_required));
222 //KdPrint((__DRIVER_NAME " tcp_length = %d, mss = %d\n", pi->tcp_length, pi->mss));
223 //KdPrint((__DRIVER_NAME " total_length = %d\n", pi->total_length));
224 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
225 //KdPrint((__DRIVER_NAME " header_extra = %d\n", header_extra));
226 if (pi->split_required)
227 {
228 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
229 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
230 //KdPrint((__DRIVER_NAME " new_ip4_length = %d\n", new_ip4_length));
231 //KdPrint((__DRIVER_NAME " this tcp_length = %d\n", tcp_length));
232 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
233 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
234 pi->tcp_seq += tcp_length;
235 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
236 /* part of the packet is already present in the header buffer for lookahead */
237 out_remaining = tcp_length - header_extra;
238 }
239 else
240 {
241 out_remaining = pi->total_length - pi->header_length;
242 }
243 //KdPrint((__DRIVER_NAME " before loop - out_remaining = %d\n", out_remaining));
245 while (out_remaining != 0)
246 {
247 ULONG in_buffer_offset;
248 ULONG in_buffer_length;
249 ULONG out_length;
251 //KdPrint((__DRIVER_NAME " in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_buffer, pi->curr_pb));
252 if (!pi->curr_buffer || !pi->curr_pb)
253 {
254 KdPrint((__DRIVER_NAME " out of buffers for packet\n"));
255 // TODO: free some stuff or we'll leak
256 return NULL;
257 }
258 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
259 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
260 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
261 //TODO: check status
262 NdisChainBufferAtBack(packet, out_buffer);
263 ref_pb(xi, pi->curr_pb);
264 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
265 if (pi->curr_mdl_offset == in_buffer_length)
266 {
267 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
268 pi->curr_pb = pi->curr_pb->next;
269 pi->curr_mdl_offset = 0;
270 }
271 out_remaining -= out_length;
272 }
273 if (pi->split_required)
274 {
275 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
276 }
277 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
278 if (header_extra > 0)
279 pi->header_length -= header_extra;
280 xi->rx_outstanding++;
281 //FUNCTION_EXIT();
282 return packet;
283 }
285 /*
286 Windows appears to insist that the checksum on received packets is correct, and won't
287 believe us when we lie about it, which happens when the packet is generated on the
288 same bridge in Dom0. Doh!
289 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
290 */
292 static BOOLEAN
293 XenNet_SumPacketData(
294 packet_info_t *pi,
295 PNDIS_PACKET packet,
296 BOOLEAN set_csum
297 )
298 {
299 USHORT i;
300 PUCHAR buffer;
301 PMDL mdl;
302 UINT total_length;
303 UINT data_length;
304 UINT buffer_length;
305 USHORT buffer_offset;
306 ULONG csum;
307 PUSHORT csum_ptr;
308 USHORT remaining;
309 USHORT ip4_length;
310 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
312 //FUNCTION_ENTER();
314 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
315 ASSERT(mdl);
317 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
318 data_length = ip4_length + XN_HDR_SIZE;
320 if ((USHORT)data_length > total_length)
321 {
322 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
323 return FALSE;
324 }
326 switch (pi->ip_proto)
327 {
328 case 6:
329 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
330 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
331 break;
332 case 17:
333 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
334 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
335 break;
336 default:
337 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
338 //FUNCTION_EXIT();
339 return FALSE; // should never happen
340 }
342 if (set_csum)
343 *csum_ptr = 0;
345 csum = 0;
346 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
347 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
348 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
350 remaining = ip4_length - pi->ip4_header_length;
352 csum += remaining;
354 csum_span = FALSE;
355 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
356 while (i < data_length)
357 {
358 /* don't include the checksum field itself in the calculation */
359 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
360 {
361 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
362 i += 2;
363 buffer_offset += 2;
364 continue;
365 }
366 if (csum_span)
367 {
368 /* the other half of the next bit */
369 ASSERT(buffer_offset == 0);
370 csum += (USHORT)buffer[buffer_offset];
371 csum_span = FALSE;
372 i += 1;
373 buffer_offset += 1;
374 }
375 else if (buffer_offset == buffer_length - 1)
376 {
377 /* deal with a buffer ending on an odd byte boundary */
378 csum += (USHORT)buffer[buffer_offset] << 8;
379 csum_span = TRUE;
380 i += 1;
381 buffer_offset += 1;
382 }
383 else
384 {
385 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
386 i += 2;
387 buffer_offset += 2;
388 }
389 if (buffer_offset == buffer_length && i < total_length)
390 {
391 NdisGetNextBuffer(mdl, &mdl);
392 if (mdl == NULL)
393 {
394 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
395 return FALSE; // should never happen
396 }
397 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
398 ASSERT(buffer_length);
399 buffer_offset = 0;
400 }
401 }
403 while (csum & 0xFFFF0000)
404 csum = (csum & 0xFFFF) + (csum >> 16);
406 if (set_csum)
407 {
408 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
409 }
410 else
411 {
412 //FUNCTION_EXIT();
413 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
414 }
415 //FUNCTION_EXIT();
416 return TRUE;
417 }
419 static ULONG
420 XenNet_MakePackets(
421 struct xennet_info *xi,
422 PLIST_ENTRY rx_packet_list
423 )
424 {
425 ULONG packet_count = 0;
426 PNDIS_PACKET packet;
427 PLIST_ENTRY entry;
428 UCHAR psh;
429 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
430 ULONG parse_result;
431 packet_info_t *pi = &xi->rxpi;
432 //PNDIS_BUFFER buffer;
433 shared_buffer_t *page_buf;
435 //FUNCTION_ENTER();
437 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
439 if (!XenNet_FilterAcceptPacket(xi, pi))
440 {
441 goto done;
442 }
444 switch (pi->ip_proto)
445 {
446 case 6: // TCP
447 if (pi->split_required)
448 break;
449 // fallthrough
450 case 17: // UDP
451 packet = XenNet_MakePacket(xi);
452 if (packet == NULL)
453 {
454 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
455 xi->stat_rx_no_buffer++;
456 packet_count = 0;
457 goto done;
458 }
459 if (parse_result == PARSE_OK)
460 {
461 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
462 packet, TcpIpChecksumPacketInfo);
463 if (pi->csum_blank || pi->data_validated)
464 {
465 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
466 {
467 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
468 {
469 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
470 }
471 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
472 {
473 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
474 }
475 if (pi->csum_blank)
476 {
477 XenNet_SumPacketData(pi, packet, TRUE);
478 }
479 }
480 else if (xi->config_csum_rx_check)
481 {
482 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
483 {
484 if (XenNet_SumPacketData(pi, packet, FALSE))
485 {
486 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
487 }
488 else
489 {
490 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
491 }
492 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
493 {
494 if (XenNet_SumPacketData(pi, packet, FALSE))
495 {
496 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
497 }
498 else
499 {
500 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
501 }
502 }
503 }
504 }
505 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
506 InsertTailList(rx_packet_list, entry);
507 packet_count = 1;
508 goto done;
509 default:
510 packet = XenNet_MakePacket(xi);
511 if (packet == NULL)
512 {
513 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
514 xi->stat_rx_no_buffer++;
515 packet_count = 0;
516 goto done;
517 }
518 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
519 InsertTailList(rx_packet_list, entry);
520 packet_count = 1;
521 goto done;
522 }
524 pi->tcp_remaining = pi->tcp_length;
526 /* we can make certain assumptions here as the following code is only for tcp4 */
527 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
528 while (pi->tcp_remaining)
529 {
530 PUCHAR header_va;
531 PMDL mdl;
532 UINT total_length;
533 UINT buffer_length;
534 packet = XenNet_MakePacket(xi);
535 if (!packet)
536 {
537 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
538 xi->stat_rx_no_buffer++;
539 break; /* we are out of memory - just drop the packets */
540 }
541 if (xi->setting_csum.V4Receive.TcpChecksum)
542 {
543 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
544 packet, TcpIpChecksumPacketInfo);
545 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
546 }
547 if (psh)
548 {
549 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
550 if (pi->tcp_remaining)
551 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
552 else
553 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
554 }
555 XenNet_SumPacketData(pi, packet, TRUE);
556 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
557 InsertTailList(rx_packet_list, entry);
558 packet_count++;
559 }
561 done:
562 page_buf = pi->first_pb;
563 while (page_buf)
564 {
565 shared_buffer_t *next_pb;
567 next_pb = page_buf->next;
568 put_pb_on_freelist(xi, page_buf);
569 page_buf = next_pb;
570 }
571 XenNet_ClearPacketInfo(pi);
572 //FUNCTION_EXIT();
573 return packet_count;
574 }
576 static BOOLEAN
577 XenNet_RxQueueDpcSynchronized(PVOID context)
578 {
579 struct xennet_info *xi = context;
581 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
583 return TRUE;
584 }
586 #define MAXIMUM_PACKETS_PER_INDICATE 32
587 /*
588 We limit the number of packets per interrupt so that acks get a chance
589 under high rx load. The DPC is immediately re-scheduled */
591 #define MAX_PACKETS_PER_INTERRUPT 64
593 // Called at DISPATCH_LEVEL
594 static VOID
595 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
596 {
597 struct xennet_info *xi = context;
598 RING_IDX cons, prod;
599 LIST_ENTRY rx_packet_list;
600 PLIST_ENTRY entry;
601 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
602 ULONG packet_count = 0;
603 struct netif_rx_response *rxrsp = NULL;
604 struct netif_extra_info *ei;
605 USHORT id;
606 int more_to_do = FALSE;
607 packet_info_t *pi = &xi->rxpi;
608 //NDIS_STATUS status;
609 shared_buffer_t *page_buf;
610 PNDIS_BUFFER buffer;
612 UNREFERENCED_PARAMETER(dpc);
613 UNREFERENCED_PARAMETER(arg1);
614 UNREFERENCED_PARAMETER(arg2);
616 //FUNCTION_ENTER();
618 if (!xi->connected)
619 return; /* a delayed DPC could let this come through... just do nothing */
621 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
623 if (xi->rx_shutting_down)
624 {
625 /* there is a chance that our Dpc had been queued just before the shutdown... */
626 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
627 return;
628 }
629 InitializeListHead(&rx_packet_list);
631 do {
632 prod = xi->rx.sring->rsp_prod;
633 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
634 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
636 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
637 {
638 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
639 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
640 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
641 xi->rx_ring_pbs[id] = 0xFFFF;
642 xi->rx_id_free++;
643 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
644 if (pi->extra_info)
645 {
646 //KdPrint((__DRIVER_NAME " processing extra info\n"));
647 put_pb_on_freelist(xi, page_buf);
648 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
649 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
650 switch (ei->type)
651 {
652 case XEN_NETIF_EXTRA_TYPE_GSO:
653 switch (ei->u.gso.type)
654 {
655 case XEN_NETIF_GSO_TYPE_TCPV4:
656 pi->mss = ei->u.gso.size;
657 //KdPrint((__DRIVER_NAME " mss = %d\n", pi->mss));
658 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
659 break;
660 default:
661 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
662 break;
663 }
664 break;
665 default:
666 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
667 break;
668 }
669 }
670 else
671 {
672 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
673 if (rxrsp->status <= 0
674 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
675 {
676 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
677 rxrsp->offset, rxrsp->status));
678 ASSERT(!pi->extra_info);
679 put_pb_on_freelist(xi, page_buf);
680 continue;
681 }
682 ASSERT(!rxrsp->offset);
683 ASSERT(rxrsp->id == id);
684 if (!pi->more_frags) // handling the packet's 1st buffer
685 {
686 if (rxrsp->flags & NETRXF_csum_blank)
687 pi->csum_blank = TRUE;
688 if (rxrsp->flags & NETRXF_data_validated)
689 pi->data_validated = TRUE;
690 }
691 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
692 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
693 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
694 buffer = page_buf->buffer;
695 NdisAdjustBufferLength(buffer, rxrsp->status);
696 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
697 if (pi->first_pb)
698 {
699 //KdPrint((__DRIVER_NAME " additional buffer\n"));
700 pi->curr_pb->next = page_buf;
701 pi->curr_pb = page_buf;
702 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
703 pi->curr_buffer = buffer;
704 }
705 else
706 {
707 pi->first_pb = page_buf;
708 pi->curr_pb = page_buf;
709 pi->first_buffer = buffer;
710 pi->curr_buffer = buffer;
711 }
712 pi->mdl_count++;
713 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
714 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
715 pi->total_length = pi->total_length + rxrsp->status;
716 }
718 /* Packet done, add it to the list */
719 if (!pi->more_frags && !pi->extra_info)
720 {
721 pi->curr_pb = pi->first_pb;
722 pi->curr_buffer = pi->first_buffer;
723 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
724 }
725 }
726 xi->rx.rsp_cons = cons;
728 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
729 break;
731 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
732 if (!more_to_do)
733 {
734 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
735 mb();
736 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
737 }
738 } while (more_to_do);
740 if (pi->more_frags || pi->extra_info)
741 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
743 /* Give netback more buffers */
744 XenNet_FillRing(xi);
746 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
747 {
748 /* fire again immediately */
749 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
750 }
752 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
753 xi->stat_rx_ok += packet_count;
755 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
757 entry = RemoveHeadList(&rx_packet_list);
758 packet_count = 0;
759 while (entry != &rx_packet_list)
760 {
761 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[0]);
762 packets[packet_count++] = packet;
763 entry = RemoveHeadList(&rx_packet_list);
764 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
765 {
766 //KdPrint((__DRIVER_NAME " Indicating\n"));
767 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
768 packet_count = 0;
769 }
770 }
771 //FUNCTION_EXIT();
772 }
774 /* called at DISPATCH_LEVEL */
775 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
776 VOID DDKAPI
777 XenNet_ReturnPacket(
778 IN NDIS_HANDLE MiniportAdapterContext,
779 IN PNDIS_PACKET Packet
780 )
781 {
782 struct xennet_info *xi = MiniportAdapterContext;
783 PNDIS_BUFFER buffer;
784 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[sizeof(LIST_ENTRY)];
786 //FUNCTION_ENTER();
788 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
790 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
792 NdisUnchainBufferAtFront(Packet, &buffer);
793 while (buffer)
794 {
795 shared_buffer_t *next_buf = page_buf->next;
796 if (!page_buf->virtual)
797 {
798 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
799 PUCHAR va;
800 UINT len;
801 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
802 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
803 NdisFreeBuffer(buffer);
804 }
805 else
806 {
807 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
808 if (buffer != page_buf->buffer)
809 NdisFreeBuffer(buffer);
810 put_pb_on_freelist(xi, page_buf);
811 }
812 NdisUnchainBufferAtFront(Packet, &buffer);
813 page_buf = next_buf;
814 }
816 put_packet_on_freelist(xi, Packet);
817 xi->rx_outstanding--;
819 if (!xi->rx_outstanding && xi->rx_shutting_down)
820 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
822 XenNet_FillRing(xi);
824 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
826 //FUNCTION_EXIT();
827 }
829 /*
830 Free all Rx buffers (on halt, for example)
831 The ring must be stopped at this point.
832 */
834 static VOID
835 XenNet_PurgeRing(struct xennet_info *xi)
836 {
837 int i;
838 for (i = 0; i < NET_RX_RING_SIZE; i++)
839 {
840 if (xi->rx_ring_pbs[i] != 0xFFFF)
841 {
842 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
843 xi->rx_ring_pbs[i] = 0xFFFF;
844 }
845 }
846 }
848 static VOID
849 XenNet_BufferFree(struct xennet_info *xi)
850 {
851 shared_buffer_t *pb;
853 XenNet_PurgeRing(xi);
855 while ((pb = get_pb_from_freelist(xi)) != NULL)
856 {
857 NdisFreeBuffer(pb->buffer);
858 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
859 pb->gref, FALSE, (ULONG)'XNRX');
860 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
861 }
862 }
864 VOID
865 XenNet_RxResumeStart(xennet_info_t *xi)
866 {
867 KIRQL old_irql;
869 FUNCTION_ENTER();
871 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
872 XenNet_PurgeRing(xi);
873 KeReleaseSpinLock(&xi->rx_lock, old_irql);
875 FUNCTION_EXIT();
876 }
878 VOID
879 XenNet_BufferAlloc(xennet_info_t *xi)
880 {
881 NDIS_STATUS status;
882 int i;
884 xi->rx_id_free = NET_RX_RING_SIZE;
885 xi->rx_outstanding = 0;
887 for (i = 0; i < NET_RX_RING_SIZE; i++)
888 {
889 xi->rx_ring_pbs[i] = 0xFFFF;
890 }
892 for (i = 0; i < RX_PAGE_BUFFERS; i++)
893 {
894 xi->rx_pbs[i].id = (USHORT)i;
895 status = NdisAllocateMemoryWithTag(&xi->rx_pbs[i].virtual, PAGE_SIZE, XENNET_POOL_TAG);
896 if (status != STATUS_SUCCESS)
897 {
898 break;
899 }
900 xi->rx_pbs[i].gref = (grant_ref_t)xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
901 (ULONG)(MmGetPhysicalAddress(xi->rx_pbs[i].virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
902 if (xi->rx_pbs[i].gref == INVALID_GRANT_REF)
903 {
904 NdisFreeMemory(xi->rx_pbs[i].virtual, PAGE_SIZE, 0);
905 break;
906 }
907 xi->rx_pbs[i].offset = (ULONG_PTR)xi->rx_pbs[i].virtual & (PAGE_SIZE - 1);
908 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
909 if (status != STATUS_SUCCESS)
910 {
911 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
912 xi->rx_pbs[i].gref, FALSE, (ULONG)'XNRX');
913 NdisFreeMemory(xi->rx_pbs[i].virtual, PAGE_SIZE, 0);
914 break;
915 }
916 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
917 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
918 }
919 if (i == 0)
920 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
921 }
924 VOID
925 XenNet_RxResumeEnd(xennet_info_t *xi)
926 {
927 KIRQL old_irql;
929 FUNCTION_ENTER();
931 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
932 //XenNet_BufferAlloc(xi);
933 XenNet_FillRing(xi);
934 KeReleaseSpinLock(&xi->rx_lock, old_irql);
936 FUNCTION_EXIT();
937 }
939 BOOLEAN
940 XenNet_RxInit(xennet_info_t *xi)
941 {
942 NDIS_STATUS status;
944 FUNCTION_ENTER();
946 xi->rx_shutting_down = FALSE;
947 KeInitializeSpinLock(&xi->rx_lock);
948 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
949 KeInitializeTimer(&xi->rx_timer);
950 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
951 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
952 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
953 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
955 XenNet_BufferAlloc(xi);
957 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
958 PROTOCOL_RESERVED_SIZE_IN_PACKET);
959 if (status != NDIS_STATUS_SUCCESS)
960 {
961 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
962 return FALSE;
963 }
965 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0,
966 MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH + sizeof(shared_buffer_t), XENNET_POOL_TAG, 0);
968 XenNet_FillRing(xi);
970 FUNCTION_EXIT();
972 return TRUE;
973 }
975 BOOLEAN
976 XenNet_RxShutdown(xennet_info_t *xi)
977 {
978 KIRQL old_irql;
980 FUNCTION_ENTER();
982 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
983 xi->rx_shutting_down = TRUE;
984 KeReleaseSpinLock(&xi->rx_lock, old_irql);
986 if (xi->config_rx_interrupt_moderation)
987 {
988 KeCancelTimer(&xi->rx_timer);
989 }
991 KeRemoveQueueDpc(&xi->rx_dpc);
992 KeFlushQueuedDpcs();
994 while (xi->rx_outstanding)
995 {
996 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
997 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
998 }
1000 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1002 XenNet_BufferFree(xi);
1004 packet_freelist_dispose(xi);
1006 NdisFreePacketPool(xi->rx_packet_pool);
1008 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1010 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1012 FUNCTION_EXIT();
1014 return TRUE;