win-pvdrivers

view xennet/xennet_rx.c @ 821:9c0c4210b778

Fix xennet build under Windows 2000
Fix xenvbd install under Windows 2000
author James Harper <james.harper@bendigoit.com.au>
date Sat Oct 16 20:03:30 2010 +1100 (2010-10-16)
parents 467005e7f509
children 7e39c337e2c3
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 /* Not really necessary but keeps PREfast happy */
24 #if (NTDDI_VERSION >= NTDDI_WINXP)
25 static KDEFERRED_ROUTINE XenNet_RxBufferCheck;
26 #endif
28 static __inline shared_buffer_t *
29 get_pb_from_freelist(struct xennet_info *xi)
30 {
31 shared_buffer_t *pb;
33 if (xi->rx_pb_free == 0)
34 {
35 //KdPrint((__DRIVER_NAME " Out of pb's\n"));
36 return NULL;
37 }
38 xi->rx_pb_free--;
40 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
41 pb->ref_count++;
42 return pb;
43 }
45 static __inline VOID
46 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
47 {
48 UNREFERENCED_PARAMETER(xi);
49 pb->ref_count++;
50 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
51 }
53 static __inline VOID
54 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
55 {
56 pb->ref_count--;
57 if (pb->ref_count == 0)
58 {
59 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
60 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
61 pb->next = NULL;
62 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
63 xi->rx_pb_free++;
64 }
65 }
67 // Called at DISPATCH_LEVEL with rx lock held
68 static NDIS_STATUS
69 XenNet_FillRing(struct xennet_info *xi)
70 {
71 unsigned short id;
72 shared_buffer_t *page_buf;
73 ULONG i, notify;
74 ULONG batch_target;
75 RING_IDX req_prod = xi->rx.req_prod_pvt;
76 netif_rx_request_t *req;
78 //FUNCTION_ENTER();
80 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
82 if (batch_target < (xi->rx_target >> 2))
83 {
84 //FUNCTION_EXIT();
85 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
86 }
88 for (i = 0; i < batch_target; i++)
89 {
90 page_buf = get_pb_from_freelist(xi);
91 if (!page_buf)
92 {
93 //KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
94 break;
95 }
96 xi->rx_id_free--;
98 /* Give to netback */
99 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
100 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
101 xi->rx_ring_pbs[id] = page_buf->id;
102 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
103 req->id = id;
104 req->gref = page_buf->gref;
105 ASSERT(req->gref != INVALID_GRANT_REF);
106 }
107 KeMemoryBarrier();
108 xi->rx.req_prod_pvt = req_prod + i;
109 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
110 if (notify)
111 {
112 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
113 }
115 //FUNCTION_EXIT();
117 return NDIS_STATUS_SUCCESS;
118 }
120 static PNDIS_PACKET
121 get_packet_from_freelist(struct xennet_info *xi)
122 {
123 NDIS_STATUS status;
124 PNDIS_PACKET packet;
126 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
128 if (!xi->rx_packet_free)
129 {
130 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
131 if (status != NDIS_STATUS_SUCCESS)
132 {
133 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
134 return NULL;
135 }
136 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
137 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
138 }
139 else
140 {
141 xi->rx_packet_free--;
142 packet = xi->rx_packet_list[xi->rx_packet_free];
143 }
144 return packet;
145 }
147 static VOID
148 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
149 {
150 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
151 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
153 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
154 {
155 //KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
156 NdisFreePacket(packet);
157 return;
158 }
159 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
160 packet, TcpIpChecksumPacketInfo);
161 csum_info->Value = 0;
162 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
163 xi->rx_packet_list[xi->rx_packet_free] = packet;
164 xi->rx_packet_free++;
165 }
167 static VOID
168 packet_freelist_dispose(struct xennet_info *xi)
169 {
170 while(xi->rx_packet_free != 0)
171 {
172 xi->rx_packet_free--;
173 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
174 }
175 }
177 static PNDIS_PACKET
178 XenNet_MakePacket(struct xennet_info *xi)
179 {
180 NDIS_STATUS status;
181 PNDIS_PACKET packet;
182 PNDIS_BUFFER out_buffer;
183 USHORT new_ip4_length;
184 PUCHAR header_va;
185 packet_info_t *pi = &xi->rxpi;
186 ULONG out_remaining;
187 ULONG tcp_length;
188 ULONG header_extra;
189 shared_buffer_t *header_buf;
191 //FUNCTION_ENTER();
193 packet = get_packet_from_freelist(xi);
194 if (packet == NULL)
195 {
196 /* buffers will be freed in MakePackets */
197 KdPrint((__DRIVER_NAME " No free packets\n"));
198 //FUNCTION_EXIT();
199 return NULL;
200 }
202 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
203 ASSERT(header_buf); // lazy
204 header_va = (PUCHAR)(header_buf + 1);
205 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
206 NdisMoveMemory(header_va, pi->header, pi->header_length);
207 //KdPrint((__DRIVER_NAME " header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead));
208 //KdPrint((__DRIVER_NAME " ip4_header_length = %d\n", pi->ip4_header_length));
209 //KdPrint((__DRIVER_NAME " tcp_header_length = %d\n", pi->tcp_header_length));
210 /* make sure we satisfy the lookahead requirement */
211 XenNet_BuildHeader(pi, header_va, max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead) + MAX_ETH_HEADER_LENGTH);
212 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
213 ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
214 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
215 ASSERT(status == STATUS_SUCCESS);
216 NdisChainBufferAtBack(packet, out_buffer);
217 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = header_buf;
218 header_buf->next = pi->curr_pb;
221 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too... maybe
223 //KdPrint((__DRIVER_NAME " split_required = %d\n", pi->split_required));
224 //KdPrint((__DRIVER_NAME " tcp_length = %d, mss = %d\n", pi->tcp_length, pi->mss));
225 //KdPrint((__DRIVER_NAME " total_length = %d\n", pi->total_length));
226 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
227 //KdPrint((__DRIVER_NAME " header_extra = %d\n", header_extra));
228 if (pi->split_required)
229 {
230 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
231 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
232 //KdPrint((__DRIVER_NAME " new_ip4_length = %d\n", new_ip4_length));
233 //KdPrint((__DRIVER_NAME " this tcp_length = %d\n", tcp_length));
234 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
235 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
236 pi->tcp_seq += tcp_length;
237 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
238 /* part of the packet is already present in the header buffer for lookahead */
239 out_remaining = tcp_length - header_extra;
240 }
241 else
242 {
243 out_remaining = pi->total_length - pi->header_length;
244 }
245 //KdPrint((__DRIVER_NAME " before loop - out_remaining = %d\n", out_remaining));
247 while (out_remaining != 0)
248 {
249 ULONG in_buffer_offset;
250 ULONG in_buffer_length;
251 ULONG out_length;
253 //KdPrint((__DRIVER_NAME " in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_buffer, pi->curr_pb));
254 if (!pi->curr_buffer || !pi->curr_pb)
255 {
256 KdPrint((__DRIVER_NAME " out of buffers for packet\n"));
257 // TODO: free some stuff or we'll leak
258 return NULL;
259 }
260 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
261 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
262 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
263 //TODO: check status
264 NdisChainBufferAtBack(packet, out_buffer);
265 ref_pb(xi, pi->curr_pb);
266 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
267 if (pi->curr_mdl_offset == in_buffer_length)
268 {
269 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
270 pi->curr_pb = pi->curr_pb->next;
271 pi->curr_mdl_offset = 0;
272 }
273 out_remaining -= out_length;
274 }
275 if (pi->split_required)
276 {
277 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
278 }
279 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
280 if (header_extra > 0)
281 pi->header_length -= header_extra;
282 xi->rx_outstanding++;
283 //FUNCTION_EXIT();
284 return packet;
285 }
287 /*
288 Windows appears to insist that the checksum on received packets is correct, and won't
289 believe us when we lie about it, which happens when the packet is generated on the
290 same bridge in Dom0. Doh!
291 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
292 */
294 static BOOLEAN
295 XenNet_SumPacketData(
296 packet_info_t *pi,
297 PNDIS_PACKET packet,
298 BOOLEAN set_csum
299 )
300 {
301 USHORT i;
302 PUCHAR buffer;
303 PMDL mdl;
304 UINT total_length;
305 UINT data_length;
306 UINT buffer_length;
307 USHORT buffer_offset;
308 ULONG csum;
309 PUSHORT csum_ptr;
310 USHORT remaining;
311 USHORT ip4_length;
312 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
314 //FUNCTION_ENTER();
316 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
317 ASSERT(mdl);
319 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
320 data_length = ip4_length + XN_HDR_SIZE;
322 if ((USHORT)data_length > total_length)
323 {
324 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
325 return FALSE;
326 }
328 switch (pi->ip_proto)
329 {
330 case 6:
331 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
332 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
333 break;
334 case 17:
335 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
336 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
337 break;
338 default:
339 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
340 //FUNCTION_EXIT();
341 return FALSE; // should never happen
342 }
344 if (set_csum)
345 *csum_ptr = 0;
347 csum = 0;
348 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
349 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
350 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
352 remaining = ip4_length - pi->ip4_header_length;
354 csum += remaining;
356 csum_span = FALSE;
357 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
358 while (i < data_length)
359 {
360 /* don't include the checksum field itself in the calculation */
361 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
362 {
363 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
364 i += 2;
365 buffer_offset += 2;
366 continue;
367 }
368 if (csum_span)
369 {
370 /* the other half of the next bit */
371 ASSERT(buffer_offset == 0);
372 csum += (USHORT)buffer[buffer_offset];
373 csum_span = FALSE;
374 i += 1;
375 buffer_offset += 1;
376 }
377 else if (buffer_offset == buffer_length - 1)
378 {
379 /* deal with a buffer ending on an odd byte boundary */
380 csum += (USHORT)buffer[buffer_offset] << 8;
381 csum_span = TRUE;
382 i += 1;
383 buffer_offset += 1;
384 }
385 else
386 {
387 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
388 i += 2;
389 buffer_offset += 2;
390 }
391 if (buffer_offset == buffer_length && i < total_length)
392 {
393 NdisGetNextBuffer(mdl, &mdl);
394 if (mdl == NULL)
395 {
396 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
397 return FALSE; // should never happen
398 }
399 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
400 ASSERT(buffer_length);
401 buffer_offset = 0;
402 }
403 }
405 while (csum & 0xFFFF0000)
406 csum = (csum & 0xFFFF) + (csum >> 16);
408 if (set_csum)
409 {
410 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
411 }
412 else
413 {
414 //FUNCTION_EXIT();
415 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
416 }
417 //FUNCTION_EXIT();
418 return TRUE;
419 }
421 static ULONG
422 XenNet_MakePackets(
423 struct xennet_info *xi,
424 PLIST_ENTRY rx_packet_list
425 )
426 {
427 ULONG packet_count = 0;
428 PNDIS_PACKET packet;
429 PLIST_ENTRY entry;
430 UCHAR psh;
431 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
432 ULONG parse_result;
433 packet_info_t *pi = &xi->rxpi;
434 //PNDIS_BUFFER buffer;
435 shared_buffer_t *page_buf;
437 //FUNCTION_ENTER();
439 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
441 if (!XenNet_FilterAcceptPacket(xi, pi))
442 {
443 goto done;
444 }
446 switch (pi->ip_proto)
447 {
448 case 6: // TCP
449 if (pi->split_required)
450 break;
451 // fallthrough
452 case 17: // UDP
453 packet = XenNet_MakePacket(xi);
454 if (packet == NULL)
455 {
456 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
457 xi->stat_rx_no_buffer++;
458 packet_count = 0;
459 goto done;
460 }
461 if (parse_result == PARSE_OK)
462 {
463 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
464 packet, TcpIpChecksumPacketInfo);
465 if (pi->csum_blank || pi->data_validated)
466 {
467 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
468 {
469 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
470 {
471 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
472 }
473 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
474 {
475 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
476 }
477 if (pi->csum_blank)
478 {
479 XenNet_SumPacketData(pi, packet, TRUE);
480 }
481 }
482 else if (xi->config_csum_rx_check)
483 {
484 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
485 {
486 if (XenNet_SumPacketData(pi, packet, FALSE))
487 {
488 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
489 }
490 else
491 {
492 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
493 }
494 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
495 {
496 if (XenNet_SumPacketData(pi, packet, FALSE))
497 {
498 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
499 }
500 else
501 {
502 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
503 }
504 }
505 }
506 }
507 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
508 InsertTailList(rx_packet_list, entry);
509 packet_count = 1;
510 goto done;
511 default:
512 packet = XenNet_MakePacket(xi);
513 if (packet == NULL)
514 {
515 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
516 xi->stat_rx_no_buffer++;
517 packet_count = 0;
518 goto done;
519 }
520 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
521 InsertTailList(rx_packet_list, entry);
522 packet_count = 1;
523 goto done;
524 }
526 pi->tcp_remaining = pi->tcp_length;
528 /* we can make certain assumptions here as the following code is only for tcp4 */
529 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
530 while (pi->tcp_remaining)
531 {
532 PUCHAR header_va;
533 PMDL mdl;
534 UINT total_length;
535 UINT buffer_length;
536 packet = XenNet_MakePacket(xi);
537 if (!packet)
538 {
539 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
540 xi->stat_rx_no_buffer++;
541 break; /* we are out of memory - just drop the packets */
542 }
543 if (xi->setting_csum.V4Receive.TcpChecksum)
544 {
545 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
546 packet, TcpIpChecksumPacketInfo);
547 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
548 }
549 if (psh)
550 {
551 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
552 if (pi->tcp_remaining)
553 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
554 else
555 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
556 }
557 XenNet_SumPacketData(pi, packet, TRUE);
558 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
559 InsertTailList(rx_packet_list, entry);
560 packet_count++;
561 }
563 done:
564 page_buf = pi->first_pb;
565 while (page_buf)
566 {
567 shared_buffer_t *next_pb;
569 next_pb = page_buf->next;
570 put_pb_on_freelist(xi, page_buf);
571 page_buf = next_pb;
572 }
573 XenNet_ClearPacketInfo(pi);
574 //FUNCTION_EXIT();
575 return packet_count;
576 }
578 static BOOLEAN
579 XenNet_RxQueueDpcSynchronized(PVOID context)
580 {
581 struct xennet_info *xi = context;
583 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
585 return TRUE;
586 }
588 #define MAXIMUM_PACKETS_PER_INDICATE 32
589 /*
590 We limit the number of packets per interrupt so that acks get a chance
591 under high rx load. The DPC is immediately re-scheduled */
593 #define MAX_PACKETS_PER_INTERRUPT 64
595 // Called at DISPATCH_LEVEL
596 static VOID
597 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
598 {
599 struct xennet_info *xi = context;
600 RING_IDX cons, prod;
601 LIST_ENTRY rx_packet_list;
602 PLIST_ENTRY entry;
603 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
604 ULONG packet_count = 0;
605 struct netif_rx_response *rxrsp = NULL;
606 struct netif_extra_info *ei;
607 USHORT id;
608 int more_to_do = FALSE;
609 packet_info_t *pi = &xi->rxpi;
610 //NDIS_STATUS status;
611 shared_buffer_t *page_buf;
612 PNDIS_BUFFER buffer;
614 UNREFERENCED_PARAMETER(dpc);
615 UNREFERENCED_PARAMETER(arg1);
616 UNREFERENCED_PARAMETER(arg2);
618 //FUNCTION_ENTER();
620 if (!xi->connected)
621 return; /* a delayed DPC could let this come through... just do nothing */
623 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
625 if (xi->rx_shutting_down)
626 {
627 /* there is a chance that our Dpc had been queued just before the shutdown... */
628 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
629 return;
630 }
631 InitializeListHead(&rx_packet_list);
633 do {
634 prod = xi->rx.sring->rsp_prod;
635 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
636 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
638 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
639 {
640 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
641 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
642 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
643 xi->rx_ring_pbs[id] = 0xFFFF;
644 xi->rx_id_free++;
645 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
646 if (pi->extra_info)
647 {
648 //KdPrint((__DRIVER_NAME " processing extra info\n"));
649 put_pb_on_freelist(xi, page_buf);
650 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
651 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
652 switch (ei->type)
653 {
654 case XEN_NETIF_EXTRA_TYPE_GSO:
655 switch (ei->u.gso.type)
656 {
657 case XEN_NETIF_GSO_TYPE_TCPV4:
658 pi->mss = ei->u.gso.size;
659 //KdPrint((__DRIVER_NAME " mss = %d\n", pi->mss));
660 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
661 break;
662 default:
663 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
664 break;
665 }
666 break;
667 default:
668 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
669 break;
670 }
671 }
672 else
673 {
674 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
675 if (rxrsp->status <= 0
676 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
677 {
678 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
679 rxrsp->offset, rxrsp->status));
680 ASSERT(!pi->extra_info);
681 put_pb_on_freelist(xi, page_buf);
682 continue;
683 }
684 ASSERT(!rxrsp->offset);
685 ASSERT(rxrsp->id == id);
686 if (!pi->more_frags) // handling the packet's 1st buffer
687 {
688 if (rxrsp->flags & NETRXF_csum_blank)
689 pi->csum_blank = TRUE;
690 if (rxrsp->flags & NETRXF_data_validated)
691 pi->data_validated = TRUE;
692 }
693 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
694 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
695 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
696 buffer = page_buf->buffer;
697 NdisAdjustBufferLength(buffer, rxrsp->status);
698 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
699 if (pi->first_pb)
700 {
701 //KdPrint((__DRIVER_NAME " additional buffer\n"));
702 pi->curr_pb->next = page_buf;
703 pi->curr_pb = page_buf;
704 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
705 pi->curr_buffer = buffer;
706 }
707 else
708 {
709 pi->first_pb = page_buf;
710 pi->curr_pb = page_buf;
711 pi->first_buffer = buffer;
712 pi->curr_buffer = buffer;
713 }
714 pi->mdl_count++;
715 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
716 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
717 pi->total_length = pi->total_length + rxrsp->status;
718 }
720 /* Packet done, add it to the list */
721 if (!pi->more_frags && !pi->extra_info)
722 {
723 pi->curr_pb = pi->first_pb;
724 pi->curr_buffer = pi->first_buffer;
725 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
726 }
727 }
728 xi->rx.rsp_cons = cons;
730 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
731 break;
733 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
734 if (!more_to_do)
735 {
736 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
737 mb();
738 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
739 }
740 } while (more_to_do);
742 if (pi->more_frags || pi->extra_info)
743 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
745 /* Give netback more buffers */
746 XenNet_FillRing(xi);
748 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
749 {
750 /* fire again immediately */
751 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
752 }
754 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
755 xi->stat_rx_ok += packet_count;
757 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
759 entry = RemoveHeadList(&rx_packet_list);
760 packet_count = 0;
761 while (entry != &rx_packet_list)
762 {
763 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[0]);
764 packets[packet_count++] = packet;
765 entry = RemoveHeadList(&rx_packet_list);
766 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
767 {
768 //KdPrint((__DRIVER_NAME " Indicating\n"));
769 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
770 packet_count = 0;
771 }
772 }
773 //FUNCTION_EXIT();
774 }
776 /* called at DISPATCH_LEVEL */
777 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
778 VOID DDKAPI
779 XenNet_ReturnPacket(
780 IN NDIS_HANDLE MiniportAdapterContext,
781 IN PNDIS_PACKET Packet
782 )
783 {
784 struct xennet_info *xi = MiniportAdapterContext;
785 PNDIS_BUFFER buffer;
786 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[sizeof(LIST_ENTRY)];
788 //FUNCTION_ENTER();
790 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
792 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
794 NdisUnchainBufferAtFront(Packet, &buffer);
795 while (buffer)
796 {
797 shared_buffer_t *next_buf = page_buf->next;
798 if (!page_buf->virtual)
799 {
800 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
801 PUCHAR va;
802 UINT len;
803 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
804 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
805 NdisFreeBuffer(buffer);
806 }
807 else
808 {
809 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
810 if (buffer != page_buf->buffer)
811 NdisFreeBuffer(buffer);
812 put_pb_on_freelist(xi, page_buf);
813 }
814 NdisUnchainBufferAtFront(Packet, &buffer);
815 page_buf = next_buf;
816 }
818 put_packet_on_freelist(xi, Packet);
819 xi->rx_outstanding--;
821 if (!xi->rx_outstanding && xi->rx_shutting_down)
822 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
824 XenNet_FillRing(xi);
826 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
828 //FUNCTION_EXIT();
829 }
831 /*
832 Free all Rx buffers (on halt, for example)
833 The ring must be stopped at this point.
834 */
836 static VOID
837 XenNet_PurgeRing(struct xennet_info *xi)
838 {
839 int i;
840 for (i = 0; i < NET_RX_RING_SIZE; i++)
841 {
842 if (xi->rx_ring_pbs[i] != 0xFFFF)
843 {
844 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
845 xi->rx_ring_pbs[i] = 0xFFFF;
846 }
847 }
848 }
850 static VOID
851 XenNet_BufferFree(struct xennet_info *xi)
852 {
853 shared_buffer_t *pb;
855 XenNet_PurgeRing(xi);
857 while ((pb = get_pb_from_freelist(xi)) != NULL)
858 {
859 NdisFreeBuffer(pb->buffer);
860 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
861 pb->gref, FALSE, (ULONG)'XNRX');
862 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
863 }
864 }
866 VOID
867 XenNet_RxResumeStart(xennet_info_t *xi)
868 {
869 KIRQL old_irql;
871 FUNCTION_ENTER();
873 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
874 XenNet_PurgeRing(xi);
875 KeReleaseSpinLock(&xi->rx_lock, old_irql);
877 FUNCTION_EXIT();
878 }
880 VOID
881 XenNet_BufferAlloc(xennet_info_t *xi)
882 {
883 NDIS_STATUS status;
884 int i;
886 xi->rx_id_free = NET_RX_RING_SIZE;
887 xi->rx_outstanding = 0;
889 for (i = 0; i < NET_RX_RING_SIZE; i++)
890 {
891 xi->rx_ring_pbs[i] = 0xFFFF;
892 }
894 for (i = 0; i < RX_PAGE_BUFFERS; i++)
895 {
896 xi->rx_pbs[i].id = (USHORT)i;
897 status = NdisAllocateMemoryWithTag(&xi->rx_pbs[i].virtual, PAGE_SIZE, XENNET_POOL_TAG);
898 if (status != STATUS_SUCCESS)
899 {
900 break;
901 }
902 xi->rx_pbs[i].gref = (grant_ref_t)xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
903 (ULONG)(MmGetPhysicalAddress(xi->rx_pbs[i].virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
904 if (xi->rx_pbs[i].gref == INVALID_GRANT_REF)
905 {
906 NdisFreeMemory(xi->rx_pbs[i].virtual, PAGE_SIZE, 0);
907 break;
908 }
909 xi->rx_pbs[i].offset = (USHORT)(ULONG_PTR)xi->rx_pbs[i].virtual & (PAGE_SIZE - 1);
910 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
911 if (status != STATUS_SUCCESS)
912 {
913 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
914 xi->rx_pbs[i].gref, FALSE, (ULONG)'XNRX');
915 NdisFreeMemory(xi->rx_pbs[i].virtual, PAGE_SIZE, 0);
916 break;
917 }
918 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
919 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
920 }
921 if (i == 0)
922 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
923 }
926 VOID
927 XenNet_RxResumeEnd(xennet_info_t *xi)
928 {
929 KIRQL old_irql;
931 FUNCTION_ENTER();
933 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
934 //XenNet_BufferAlloc(xi);
935 XenNet_FillRing(xi);
936 KeReleaseSpinLock(&xi->rx_lock, old_irql);
938 FUNCTION_EXIT();
939 }
941 BOOLEAN
942 XenNet_RxInit(xennet_info_t *xi)
943 {
944 NDIS_STATUS status;
946 FUNCTION_ENTER();
948 xi->rx_shutting_down = FALSE;
949 KeInitializeSpinLock(&xi->rx_lock);
950 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
951 KeInitializeTimer(&xi->rx_timer);
952 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
953 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
954 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
955 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
957 XenNet_BufferAlloc(xi);
959 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
960 PROTOCOL_RESERVED_SIZE_IN_PACKET);
961 if (status != NDIS_STATUS_SUCCESS)
962 {
963 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
964 return FALSE;
965 }
967 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0,
968 MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH + sizeof(shared_buffer_t), XENNET_POOL_TAG, 0);
970 XenNet_FillRing(xi);
972 FUNCTION_EXIT();
974 return TRUE;
975 }
977 BOOLEAN
978 XenNet_RxShutdown(xennet_info_t *xi)
979 {
980 KIRQL old_irql;
982 FUNCTION_ENTER();
984 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
985 xi->rx_shutting_down = TRUE;
986 KeReleaseSpinLock(&xi->rx_lock, old_irql);
988 if (xi->config_rx_interrupt_moderation)
989 {
990 KeCancelTimer(&xi->rx_timer);
991 }
993 KeRemoveQueueDpc(&xi->rx_dpc);
994 #if (NTDDI_VERSION >= NTDDI_WINXP)
995 KeFlushQueuedDpcs();
996 #endif
998 while (xi->rx_outstanding)
999 {
1000 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1001 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1004 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1006 XenNet_BufferFree(xi);
1008 packet_freelist_dispose(xi);
1010 NdisFreePacketPool(xi->rx_packet_pool);
1012 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1014 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1016 FUNCTION_EXIT();
1018 return TRUE;