win-pvdrivers

view xennet/xennet_rx.c @ 866:c3e9661e4f7e

Handle low memory conditions better
author James Harper <james.harper@bendigoit.com.au>
date Mon Feb 28 10:04:55 2011 +1100 (2011-02-28)
parents 7e39c337e2c3
children 8d7560d67376
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 /* Not really necessary but keeps PREfast happy */
24 #if (NTDDI_VERSION >= NTDDI_WINXP)
25 static KDEFERRED_ROUTINE XenNet_RxBufferCheck;
26 #endif
28 static __inline shared_buffer_t *
29 get_pb_from_freelist(struct xennet_info *xi)
30 {
31 shared_buffer_t *pb;
33 if (xi->rx_pb_free == 0)
34 {
35 //KdPrint((__DRIVER_NAME " Out of pb's\n"));
36 return NULL;
37 }
38 xi->rx_pb_free--;
40 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
41 pb->ref_count++;
42 return pb;
43 }
45 static __inline VOID
46 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
47 {
48 UNREFERENCED_PARAMETER(xi);
49 pb->ref_count++;
50 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
51 }
53 static __inline VOID
54 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
55 {
56 pb->ref_count--;
57 if (pb->ref_count == 0)
58 {
59 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
60 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
61 pb->next = NULL;
62 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
63 xi->rx_pb_free++;
64 }
65 }
67 // Called at DISPATCH_LEVEL with rx lock held
68 static NDIS_STATUS
69 XenNet_FillRing(struct xennet_info *xi)
70 {
71 unsigned short id;
72 shared_buffer_t *page_buf;
73 ULONG i, notify;
74 ULONG batch_target;
75 RING_IDX req_prod = xi->rx.req_prod_pvt;
76 netif_rx_request_t *req;
78 //FUNCTION_ENTER();
80 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
82 if (batch_target < (xi->rx_target >> 2))
83 {
84 //FUNCTION_EXIT();
85 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
86 }
88 for (i = 0; i < batch_target; i++)
89 {
90 page_buf = get_pb_from_freelist(xi);
91 if (!page_buf)
92 {
93 //KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
94 break;
95 }
96 xi->rx_id_free--;
98 /* Give to netback */
99 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
100 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
101 xi->rx_ring_pbs[id] = page_buf->id;
102 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
103 req->id = id;
104 req->gref = page_buf->gref;
105 ASSERT(req->gref != INVALID_GRANT_REF);
106 }
107 KeMemoryBarrier();
108 xi->rx.req_prod_pvt = req_prod + i;
109 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
110 if (notify)
111 {
112 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
113 }
115 //FUNCTION_EXIT();
117 return NDIS_STATUS_SUCCESS;
118 }
120 static PNDIS_PACKET
121 get_packet_from_freelist(struct xennet_info *xi)
122 {
123 NDIS_STATUS status;
124 PNDIS_PACKET packet;
126 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
128 if (!xi->rx_packet_free)
129 {
130 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
131 if (status != NDIS_STATUS_SUCCESS)
132 {
133 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
134 return NULL;
135 }
136 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
137 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
138 }
139 else
140 {
141 xi->rx_packet_free--;
142 packet = xi->rx_packet_list[xi->rx_packet_free];
143 }
144 return packet;
145 }
147 static VOID
148 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
149 {
150 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
151 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
153 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
154 {
155 //KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
156 NdisFreePacket(packet);
157 return;
158 }
159 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
160 packet, TcpIpChecksumPacketInfo);
161 csum_info->Value = 0;
162 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
163 xi->rx_packet_list[xi->rx_packet_free] = packet;
164 xi->rx_packet_free++;
165 }
167 static VOID
168 packet_freelist_dispose(struct xennet_info *xi)
169 {
170 while(xi->rx_packet_free != 0)
171 {
172 xi->rx_packet_free--;
173 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
174 }
175 }
177 static PNDIS_PACKET
178 XenNet_MakePacket(struct xennet_info *xi)
179 {
180 NDIS_STATUS status;
181 PNDIS_PACKET packet;
182 PNDIS_BUFFER out_buffer;
183 USHORT new_ip4_length;
184 PUCHAR header_va;
185 packet_info_t *pi = &xi->rxpi;
186 ULONG out_remaining;
187 ULONG tcp_length;
188 ULONG header_extra;
189 shared_buffer_t *header_buf;
191 //FUNCTION_ENTER();
193 packet = get_packet_from_freelist(xi);
194 if (packet == NULL)
195 {
196 /* buffers will be freed in MakePackets */
197 KdPrint((__DRIVER_NAME " No free packets\n"));
198 //FUNCTION_EXIT();
199 return NULL;
200 }
202 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
203 if (!header_buf)
204 {
205 KdPrint((__DRIVER_NAME " No free header buffers\n"));
206 put_packet_on_freelist(xi, packet);
207 return NULL;
208 }
209 header_va = (PUCHAR)(header_buf + 1);
210 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
211 NdisMoveMemory(header_va, pi->header, pi->header_length);
212 //KdPrint((__DRIVER_NAME " header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead));
213 //KdPrint((__DRIVER_NAME " ip4_header_length = %d\n", pi->ip4_header_length));
214 //KdPrint((__DRIVER_NAME " tcp_header_length = %d\n", pi->tcp_header_length));
215 /* make sure we satisfy the lookahead requirement */
216 XenNet_BuildHeader(pi, header_va, max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead) + MAX_ETH_HEADER_LENGTH);
217 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
218 ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
219 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
220 if (status != STATUS_SUCCESS)
221 {
222 KdPrint((__DRIVER_NAME " No free header buffers\n"));
223 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, header_buf);
224 put_packet_on_freelist(xi, packet);
225 return NULL;
226 }
227 NdisChainBufferAtBack(packet, out_buffer);
228 *(shared_buffer_t **)&packet->MiniportReservedEx[0] = header_buf;
229 header_buf->next = pi->curr_pb;
231 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too... maybe
233 //KdPrint((__DRIVER_NAME " split_required = %d\n", pi->split_required));
234 //KdPrint((__DRIVER_NAME " tcp_length = %d, mss = %d\n", pi->tcp_length, pi->mss));
235 //KdPrint((__DRIVER_NAME " total_length = %d\n", pi->total_length));
236 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
237 //KdPrint((__DRIVER_NAME " header_extra = %d\n", header_extra));
238 if (pi->split_required)
239 {
240 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
241 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
242 //KdPrint((__DRIVER_NAME " new_ip4_length = %d\n", new_ip4_length));
243 //KdPrint((__DRIVER_NAME " this tcp_length = %d\n", tcp_length));
244 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
245 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
246 pi->tcp_seq += tcp_length;
247 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
248 /* part of the packet is already present in the header buffer for lookahead */
249 out_remaining = tcp_length - header_extra;
250 }
251 else
252 {
253 out_remaining = pi->total_length - pi->header_length;
254 }
255 //KdPrint((__DRIVER_NAME " before loop - out_remaining = %d\n", out_remaining));
257 while (out_remaining != 0)
258 {
259 ULONG in_buffer_offset;
260 ULONG in_buffer_length;
261 ULONG out_length;
263 //KdPrint((__DRIVER_NAME " in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_buffer, pi->curr_pb));
264 if (!pi->curr_buffer || !pi->curr_pb)
265 {
266 KdPrint((__DRIVER_NAME " out of buffers for packet\n"));
267 // TODO: free some stuff or we'll leak
268 return NULL;
269 }
270 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
271 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
272 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
273 //TODO: check status
274 NdisChainBufferAtBack(packet, out_buffer);
275 ref_pb(xi, pi->curr_pb);
276 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
277 if (pi->curr_mdl_offset == in_buffer_length)
278 {
279 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
280 pi->curr_pb = pi->curr_pb->next;
281 pi->curr_mdl_offset = 0;
282 }
283 out_remaining -= out_length;
284 }
285 if (pi->split_required)
286 {
287 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
288 }
289 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
290 if (header_extra > 0)
291 pi->header_length -= header_extra;
292 xi->rx_outstanding++;
293 //FUNCTION_EXIT();
294 return packet;
295 }
297 /*
298 Windows appears to insist that the checksum on received packets is correct, and won't
299 believe us when we lie about it, which happens when the packet is generated on the
300 same bridge in Dom0. Doh!
301 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
302 */
304 static BOOLEAN
305 XenNet_SumPacketData(
306 packet_info_t *pi,
307 PNDIS_PACKET packet,
308 BOOLEAN set_csum
309 )
310 {
311 USHORT i;
312 PUCHAR buffer;
313 PMDL mdl;
314 UINT total_length;
315 UINT data_length;
316 UINT buffer_length;
317 USHORT buffer_offset;
318 ULONG csum;
319 PUSHORT csum_ptr;
320 USHORT remaining;
321 USHORT ip4_length;
322 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
324 //FUNCTION_ENTER();
326 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
327 ASSERT(mdl);
329 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
330 data_length = ip4_length + XN_HDR_SIZE;
332 if ((USHORT)data_length > total_length)
333 {
334 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
335 return FALSE;
336 }
338 switch (pi->ip_proto)
339 {
340 case 6:
341 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
342 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
343 break;
344 case 17:
345 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
346 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
347 break;
348 default:
349 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
350 //FUNCTION_EXIT();
351 return FALSE; // should never happen
352 }
354 if (set_csum)
355 *csum_ptr = 0;
357 csum = 0;
358 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
359 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
360 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
362 remaining = ip4_length - pi->ip4_header_length;
364 csum += remaining;
366 csum_span = FALSE;
367 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
368 while (i < data_length)
369 {
370 /* don't include the checksum field itself in the calculation */
371 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
372 {
373 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
374 i += 2;
375 buffer_offset += 2;
376 continue;
377 }
378 if (csum_span)
379 {
380 /* the other half of the next bit */
381 ASSERT(buffer_offset == 0);
382 csum += (USHORT)buffer[buffer_offset];
383 csum_span = FALSE;
384 i += 1;
385 buffer_offset += 1;
386 }
387 else if (buffer_offset == buffer_length - 1)
388 {
389 /* deal with a buffer ending on an odd byte boundary */
390 csum += (USHORT)buffer[buffer_offset] << 8;
391 csum_span = TRUE;
392 i += 1;
393 buffer_offset += 1;
394 }
395 else
396 {
397 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
398 i += 2;
399 buffer_offset += 2;
400 }
401 if (buffer_offset == buffer_length && i < total_length)
402 {
403 NdisGetNextBuffer(mdl, &mdl);
404 if (mdl == NULL)
405 {
406 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
407 return FALSE; // should never happen
408 }
409 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
410 ASSERT(buffer_length);
411 buffer_offset = 0;
412 }
413 }
415 while (csum & 0xFFFF0000)
416 csum = (csum & 0xFFFF) + (csum >> 16);
418 if (set_csum)
419 {
420 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
421 }
422 else
423 {
424 //FUNCTION_EXIT();
425 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
426 }
427 //FUNCTION_EXIT();
428 return TRUE;
429 }
431 static ULONG
432 XenNet_MakePackets(
433 struct xennet_info *xi,
434 PLIST_ENTRY rx_packet_list
435 )
436 {
437 ULONG packet_count = 0;
438 PNDIS_PACKET packet;
439 PLIST_ENTRY entry;
440 UCHAR psh;
441 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
442 ULONG parse_result;
443 packet_info_t *pi = &xi->rxpi;
444 //PNDIS_BUFFER buffer;
445 shared_buffer_t *page_buf;
447 //FUNCTION_ENTER();
449 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
451 if (!XenNet_FilterAcceptPacket(xi, pi))
452 {
453 goto done;
454 }
456 switch (pi->ip_proto)
457 {
458 case 6: // TCP
459 if (pi->split_required)
460 break;
461 // fallthrough
462 case 17: // UDP
463 packet = XenNet_MakePacket(xi);
464 if (packet == NULL)
465 {
466 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
467 xi->stat_rx_no_buffer++;
468 packet_count = 0;
469 goto done;
470 }
471 if (parse_result == PARSE_OK)
472 {
473 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
474 packet, TcpIpChecksumPacketInfo);
475 if (pi->csum_blank || pi->data_validated)
476 {
477 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
478 {
479 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
480 {
481 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
482 }
483 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
484 {
485 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
486 }
487 if (pi->csum_blank)
488 {
489 XenNet_SumPacketData(pi, packet, TRUE);
490 }
491 }
492 else if (xi->config_csum_rx_check)
493 {
494 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
495 {
496 if (XenNet_SumPacketData(pi, packet, FALSE))
497 {
498 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
499 }
500 else
501 {
502 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
503 }
504 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
505 {
506 if (XenNet_SumPacketData(pi, packet, FALSE))
507 {
508 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
509 }
510 else
511 {
512 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
513 }
514 }
515 }
516 }
517 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
518 InsertTailList(rx_packet_list, entry);
519 packet_count = 1;
520 goto done;
521 default:
522 packet = XenNet_MakePacket(xi);
523 if (packet == NULL)
524 {
525 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
526 xi->stat_rx_no_buffer++;
527 packet_count = 0;
528 goto done;
529 }
530 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
531 InsertTailList(rx_packet_list, entry);
532 packet_count = 1;
533 goto done;
534 }
536 pi->tcp_remaining = pi->tcp_length;
538 /* we can make certain assumptions here as the following code is only for tcp4 */
539 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
540 while (pi->tcp_remaining)
541 {
542 PUCHAR header_va;
543 PMDL mdl;
544 UINT total_length;
545 UINT buffer_length;
546 packet = XenNet_MakePacket(xi);
547 if (!packet)
548 {
549 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
550 xi->stat_rx_no_buffer++;
551 break; /* we are out of memory - just drop the packets */
552 }
553 if (xi->setting_csum.V4Receive.TcpChecksum)
554 {
555 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
556 packet, TcpIpChecksumPacketInfo);
557 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
558 }
559 if (psh)
560 {
561 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
562 if (pi->tcp_remaining)
563 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
564 else
565 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
566 }
567 XenNet_SumPacketData(pi, packet, TRUE);
568 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
569 InsertTailList(rx_packet_list, entry);
570 packet_count++;
571 }
573 done:
574 page_buf = pi->first_pb;
575 while (page_buf)
576 {
577 shared_buffer_t *next_pb;
579 next_pb = page_buf->next;
580 put_pb_on_freelist(xi, page_buf);
581 page_buf = next_pb;
582 }
583 XenNet_ClearPacketInfo(pi);
584 //FUNCTION_EXIT();
585 return packet_count;
586 }
588 static BOOLEAN
589 XenNet_RxQueueDpcSynchronized(PVOID context)
590 {
591 struct xennet_info *xi = context;
593 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
595 return TRUE;
596 }
598 #define MAXIMUM_PACKETS_PER_INDICATE 32
599 /*
600 We limit the number of packets per interrupt so that acks get a chance
601 under high rx load. The DPC is immediately re-scheduled */
603 #define MAX_PACKETS_PER_INTERRUPT 64
605 // Called at DISPATCH_LEVEL
606 static VOID
607 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
608 {
609 struct xennet_info *xi = context;
610 RING_IDX cons, prod;
611 LIST_ENTRY rx_packet_list;
612 PLIST_ENTRY entry;
613 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
614 ULONG packet_count = 0;
615 struct netif_rx_response *rxrsp = NULL;
616 struct netif_extra_info *ei;
617 USHORT id;
618 int more_to_do = FALSE;
619 packet_info_t *pi = &xi->rxpi;
620 //NDIS_STATUS status;
621 shared_buffer_t *page_buf;
622 PNDIS_BUFFER buffer;
624 UNREFERENCED_PARAMETER(dpc);
625 UNREFERENCED_PARAMETER(arg1);
626 UNREFERENCED_PARAMETER(arg2);
628 //FUNCTION_ENTER();
630 if (!xi->connected)
631 return; /* a delayed DPC could let this come through... just do nothing */
633 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
635 if (xi->rx_shutting_down)
636 {
637 /* there is a chance that our Dpc had been queued just before the shutdown... */
638 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
639 return;
640 }
641 InitializeListHead(&rx_packet_list);
643 do {
644 prod = xi->rx.sring->rsp_prod;
645 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
646 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
648 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
649 {
650 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
651 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
652 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
653 xi->rx_ring_pbs[id] = 0xFFFF;
654 xi->rx_id_free++;
655 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
656 if (pi->extra_info)
657 {
658 //KdPrint((__DRIVER_NAME " processing extra info\n"));
659 put_pb_on_freelist(xi, page_buf);
660 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
661 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
662 switch (ei->type)
663 {
664 case XEN_NETIF_EXTRA_TYPE_GSO:
665 switch (ei->u.gso.type)
666 {
667 case XEN_NETIF_GSO_TYPE_TCPV4:
668 pi->mss = ei->u.gso.size;
669 //KdPrint((__DRIVER_NAME " mss = %d\n", pi->mss));
670 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
671 break;
672 default:
673 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
674 break;
675 }
676 break;
677 default:
678 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
679 break;
680 }
681 }
682 else
683 {
684 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
685 if (rxrsp->status <= 0
686 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
687 {
688 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
689 rxrsp->offset, rxrsp->status));
690 ASSERT(!pi->extra_info);
691 put_pb_on_freelist(xi, page_buf);
692 continue;
693 }
694 ASSERT(!rxrsp->offset);
695 ASSERT(rxrsp->id == id);
696 if (!pi->more_frags) // handling the packet's 1st buffer
697 {
698 if (rxrsp->flags & NETRXF_csum_blank)
699 pi->csum_blank = TRUE;
700 if (rxrsp->flags & NETRXF_data_validated)
701 pi->data_validated = TRUE;
702 }
703 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
704 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
705 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
706 buffer = page_buf->buffer;
707 NdisAdjustBufferLength(buffer, rxrsp->status);
708 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
709 if (pi->first_pb)
710 {
711 //KdPrint((__DRIVER_NAME " additional buffer\n"));
712 pi->curr_pb->next = page_buf;
713 pi->curr_pb = page_buf;
714 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
715 pi->curr_buffer = buffer;
716 }
717 else
718 {
719 pi->first_pb = page_buf;
720 pi->curr_pb = page_buf;
721 pi->first_buffer = buffer;
722 pi->curr_buffer = buffer;
723 }
724 pi->mdl_count++;
725 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
726 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
727 pi->total_length = pi->total_length + rxrsp->status;
728 }
730 /* Packet done, add it to the list */
731 if (!pi->more_frags && !pi->extra_info)
732 {
733 pi->curr_pb = pi->first_pb;
734 pi->curr_buffer = pi->first_buffer;
735 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
736 }
737 }
738 xi->rx.rsp_cons = cons;
740 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
741 break;
743 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
744 if (!more_to_do)
745 {
746 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
747 mb();
748 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
749 }
750 } while (more_to_do);
752 if (pi->more_frags || pi->extra_info)
753 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
755 /* Give netback more buffers */
756 XenNet_FillRing(xi);
758 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
759 {
760 /* fire again immediately */
761 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
762 }
764 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
765 xi->stat_rx_ok += packet_count;
767 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
769 entry = RemoveHeadList(&rx_packet_list);
770 packet_count = 0;
771 while (entry != &rx_packet_list)
772 {
773 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
774 packets[packet_count++] = packet;
775 entry = RemoveHeadList(&rx_packet_list);
776 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
777 {
778 //KdPrint((__DRIVER_NAME " Indicating\n"));
779 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
780 packet_count = 0;
781 }
782 }
783 //FUNCTION_EXIT();
784 }
786 /* called at DISPATCH_LEVEL */
787 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
788 VOID DDKAPI
789 XenNet_ReturnPacket(
790 IN NDIS_HANDLE MiniportAdapterContext,
791 IN PNDIS_PACKET Packet
792 )
793 {
794 struct xennet_info *xi = MiniportAdapterContext;
795 PNDIS_BUFFER buffer;
796 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[0];
798 //FUNCTION_ENTER();
800 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
802 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
804 NdisUnchainBufferAtFront(Packet, &buffer);
805 while (buffer)
806 {
807 shared_buffer_t *next_buf;
808 ASSERT(page_buf);
809 next_buf = page_buf->next;
810 if (!page_buf->virtual)
811 {
812 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
813 PUCHAR va;
814 UINT len;
815 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
816 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
817 NdisFreeBuffer(buffer);
818 }
819 else
820 {
821 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
822 if (buffer != page_buf->buffer)
823 NdisFreeBuffer(buffer);
824 put_pb_on_freelist(xi, page_buf);
825 }
826 NdisUnchainBufferAtFront(Packet, &buffer);
827 page_buf = next_buf;
828 }
830 put_packet_on_freelist(xi, Packet);
831 xi->rx_outstanding--;
833 if (!xi->rx_outstanding && xi->rx_shutting_down)
834 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
836 XenNet_FillRing(xi);
838 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
840 //FUNCTION_EXIT();
841 }
843 /*
844 Free all Rx buffers (on halt, for example)
845 The ring must be stopped at this point.
846 */
848 static VOID
849 XenNet_PurgeRing(struct xennet_info *xi)
850 {
851 int i;
852 for (i = 0; i < NET_RX_RING_SIZE; i++)
853 {
854 if (xi->rx_ring_pbs[i] != 0xFFFF)
855 {
856 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
857 xi->rx_ring_pbs[i] = 0xFFFF;
858 }
859 }
860 }
862 static VOID
863 XenNet_BufferFree(struct xennet_info *xi)
864 {
865 shared_buffer_t *pb;
867 XenNet_PurgeRing(xi);
869 while ((pb = get_pb_from_freelist(xi)) != NULL)
870 {
871 NdisFreeBuffer(pb->buffer);
872 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
873 pb->gref, FALSE, (ULONG)'XNRX');
874 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
875 }
876 }
878 VOID
879 XenNet_RxResumeStart(xennet_info_t *xi)
880 {
881 KIRQL old_irql;
883 FUNCTION_ENTER();
885 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
886 XenNet_PurgeRing(xi);
887 KeReleaseSpinLock(&xi->rx_lock, old_irql);
889 FUNCTION_EXIT();
890 }
892 VOID
893 XenNet_BufferAlloc(xennet_info_t *xi)
894 {
895 NDIS_STATUS status;
896 int i;
898 xi->rx_id_free = NET_RX_RING_SIZE;
899 xi->rx_outstanding = 0;
901 for (i = 0; i < NET_RX_RING_SIZE; i++)
902 {
903 xi->rx_ring_pbs[i] = 0xFFFF;
904 }
906 for (i = 0; i < RX_PAGE_BUFFERS; i++)
907 {
908 xi->rx_pbs[i].id = (USHORT)i;
909 status = NdisAllocateMemoryWithTag(&xi->rx_pbs[i].virtual, PAGE_SIZE, XENNET_POOL_TAG);
910 if (status != STATUS_SUCCESS)
911 {
912 break;
913 }
914 xi->rx_pbs[i].gref = (grant_ref_t)xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
915 (ULONG)(MmGetPhysicalAddress(xi->rx_pbs[i].virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
916 if (xi->rx_pbs[i].gref == INVALID_GRANT_REF)
917 {
918 NdisFreeMemory(xi->rx_pbs[i].virtual, PAGE_SIZE, 0);
919 break;
920 }
921 xi->rx_pbs[i].offset = (USHORT)(ULONG_PTR)xi->rx_pbs[i].virtual & (PAGE_SIZE - 1);
922 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
923 if (status != STATUS_SUCCESS)
924 {
925 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
926 xi->rx_pbs[i].gref, FALSE, (ULONG)'XNRX');
927 NdisFreeMemory(xi->rx_pbs[i].virtual, PAGE_SIZE, 0);
928 break;
929 }
930 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
931 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
932 }
933 if (i == 0)
934 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
935 }
938 VOID
939 XenNet_RxResumeEnd(xennet_info_t *xi)
940 {
941 KIRQL old_irql;
943 FUNCTION_ENTER();
945 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
946 //XenNet_BufferAlloc(xi);
947 XenNet_FillRing(xi);
948 KeReleaseSpinLock(&xi->rx_lock, old_irql);
950 FUNCTION_EXIT();
951 }
953 BOOLEAN
954 XenNet_RxInit(xennet_info_t *xi)
955 {
956 NDIS_STATUS status;
958 FUNCTION_ENTER();
960 xi->rx_shutting_down = FALSE;
961 KeInitializeSpinLock(&xi->rx_lock);
962 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
963 KeInitializeTimer(&xi->rx_timer);
964 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
965 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
966 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
967 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
969 XenNet_BufferAlloc(xi);
971 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
972 PROTOCOL_RESERVED_SIZE_IN_PACKET);
973 if (status != NDIS_STATUS_SUCCESS)
974 {
975 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
976 return FALSE;
977 }
979 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0,
980 MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH + sizeof(shared_buffer_t), XENNET_POOL_TAG, 0);
982 XenNet_FillRing(xi);
984 FUNCTION_EXIT();
986 return TRUE;
987 }
989 BOOLEAN
990 XenNet_RxShutdown(xennet_info_t *xi)
991 {
992 KIRQL old_irql;
994 FUNCTION_ENTER();
996 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
997 xi->rx_shutting_down = TRUE;
998 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1000 if (xi->config_rx_interrupt_moderation)
1002 KeCancelTimer(&xi->rx_timer);
1005 KeRemoveQueueDpc(&xi->rx_dpc);
1006 #if (NTDDI_VERSION >= NTDDI_WINXP)
1007 KeFlushQueuedDpcs();
1008 #endif
1010 while (xi->rx_outstanding)
1012 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1013 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1016 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1018 XenNet_BufferFree(xi);
1020 packet_freelist_dispose(xi);
1022 NdisFreePacketPool(xi->rx_packet_pool);
1024 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1026 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1028 FUNCTION_EXIT();
1030 return TRUE;