win-pvdrivers

view xennet/xennet_rx.c @ 783:644e5ddb1b47

Handle SCSI INQUIRY command better in xenvbd
author James Harper <james.harper@bendigoit.com.au>
date Mon Feb 15 20:53:57 2010 +1100 (2010-02-15)
parents e0cfc7938e06
children 467005e7f509
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet.h"
23 /* Not really necessary but keeps PREfast happy */
24 static KDEFERRED_ROUTINE XenNet_RxBufferCheck;
26 static __inline shared_buffer_t *
27 get_pb_from_freelist(struct xennet_info *xi)
28 {
29 shared_buffer_t *pb;
31 if (xi->rx_pb_free == 0)
32 {
33 //KdPrint((__DRIVER_NAME " Out of pb's\n"));
34 return NULL;
35 }
36 xi->rx_pb_free--;
38 pb = &xi->rx_pbs[xi->rx_pb_list[xi->rx_pb_free]];
39 pb->ref_count++;
40 return pb;
41 }
43 static __inline VOID
44 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
45 {
46 UNREFERENCED_PARAMETER(xi);
47 pb->ref_count++;
48 //KdPrint((__DRIVER_NAME " incremented pb %p ref to %d\n", pb, pb->ref_count));
49 }
51 static __inline VOID
52 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
53 {
54 pb->ref_count--;
55 if (pb->ref_count == 0)
56 {
57 NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
58 NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
59 pb->next = NULL;
60 xi->rx_pb_list[xi->rx_pb_free] = pb->id;
61 xi->rx_pb_free++;
62 }
63 }
65 // Called at DISPATCH_LEVEL with rx lock held
66 static NDIS_STATUS
67 XenNet_FillRing(struct xennet_info *xi)
68 {
69 unsigned short id;
70 shared_buffer_t *page_buf;
71 ULONG i, notify;
72 ULONG batch_target;
73 RING_IDX req_prod = xi->rx.req_prod_pvt;
74 netif_rx_request_t *req;
76 //FUNCTION_ENTER();
78 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
80 if (batch_target < (xi->rx_target >> 2))
81 {
82 //FUNCTION_EXIT();
83 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
84 }
86 for (i = 0; i < batch_target; i++)
87 {
88 page_buf = get_pb_from_freelist(xi);
89 if (!page_buf)
90 {
91 //KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
92 break;
93 }
94 xi->rx_id_free--;
96 /* Give to netback */
97 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
98 ASSERT(xi->rx_ring_pbs[id] == (USHORT)0xFFFF);
99 xi->rx_ring_pbs[id] = page_buf->id;
100 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
101 req->id = id;
102 req->gref = page_buf->gref;
103 ASSERT(req->gref != INVALID_GRANT_REF);
104 }
105 KeMemoryBarrier();
106 xi->rx.req_prod_pvt = req_prod + i;
107 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
108 if (notify)
109 {
110 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
111 }
113 //FUNCTION_EXIT();
115 return NDIS_STATUS_SUCCESS;
116 }
118 static PNDIS_PACKET
119 get_packet_from_freelist(struct xennet_info *xi)
120 {
121 NDIS_STATUS status;
122 PNDIS_PACKET packet;
124 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
126 if (!xi->rx_packet_free)
127 {
128 NdisAllocatePacket(&status, &packet, xi->rx_packet_pool);
129 if (status != NDIS_STATUS_SUCCESS)
130 {
131 KdPrint((__DRIVER_NAME " cannot allocate packet\n"));
132 return NULL;
133 }
134 NDIS_SET_PACKET_HEADER_SIZE(packet, XN_HDR_SIZE);
135 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
136 }
137 else
138 {
139 xi->rx_packet_free--;
140 packet = xi->rx_packet_list[xi->rx_packet_free];
141 }
142 return packet;
143 }
145 static VOID
146 put_packet_on_freelist(struct xennet_info *xi, PNDIS_PACKET packet)
147 {
148 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
149 //ASSERT(!KeTestSpinLock(&xi->rx_lock));
151 if (xi->rx_packet_free == NET_RX_RING_SIZE * 2)
152 {
153 //KdPrint((__DRIVER_NAME " packet free list full - releasing packet\n"));
154 NdisFreePacket(packet);
155 return;
156 }
157 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
158 packet, TcpIpChecksumPacketInfo);
159 csum_info->Value = 0;
160 NdisZeroMemory(packet->MiniportReservedEx, sizeof(packet->MiniportReservedEx));
161 xi->rx_packet_list[xi->rx_packet_free] = packet;
162 xi->rx_packet_free++;
163 }
165 static VOID
166 packet_freelist_dispose(struct xennet_info *xi)
167 {
168 while(xi->rx_packet_free != 0)
169 {
170 xi->rx_packet_free--;
171 NdisFreePacket(xi->rx_packet_list[xi->rx_packet_free]);
172 }
173 }
175 static PNDIS_PACKET
176 XenNet_MakePacket(struct xennet_info *xi)
177 {
178 NDIS_STATUS status;
179 PNDIS_PACKET packet;
180 PNDIS_BUFFER out_buffer;
181 USHORT new_ip4_length;
182 PUCHAR header_va;
183 packet_info_t *pi = &xi->rxpi;
184 ULONG out_remaining;
185 ULONG tcp_length;
186 ULONG header_extra;
187 shared_buffer_t *header_buf;
189 //FUNCTION_ENTER();
191 packet = get_packet_from_freelist(xi);
192 if (packet == NULL)
193 {
194 /* buffers will be freed in MakePackets */
195 KdPrint((__DRIVER_NAME " No free packets\n"));
196 //FUNCTION_EXIT();
197 return NULL;
198 }
200 header_buf = NdisAllocateFromNPagedLookasideList(&xi->rx_lookaside_list);
201 ASSERT(header_buf); // lazy
202 header_va = (PUCHAR)(header_buf + 1);
203 NdisZeroMemory(header_buf, sizeof(shared_buffer_t));
204 NdisMoveMemory(header_va, pi->header, pi->header_length);
205 //KdPrint((__DRIVER_NAME " header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead));
206 //KdPrint((__DRIVER_NAME " ip4_header_length = %d\n", pi->ip4_header_length));
207 //KdPrint((__DRIVER_NAME " tcp_header_length = %d\n", pi->tcp_header_length));
208 /* make sure we satisfy the lookahead requirement */
209 XenNet_BuildHeader(pi, header_va, max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead) + MAX_ETH_HEADER_LENGTH);
210 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
211 ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
212 NdisAllocateBuffer(&status, &out_buffer, xi->rx_buffer_pool, header_va, pi->header_length);
213 ASSERT(status == STATUS_SUCCESS);
214 NdisChainBufferAtBack(packet, out_buffer);
215 *(shared_buffer_t **)&packet->MiniportReservedEx[sizeof(LIST_ENTRY)] = header_buf;
216 header_buf->next = pi->curr_pb;
219 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too... maybe
221 //KdPrint((__DRIVER_NAME " split_required = %d\n", pi->split_required));
222 //KdPrint((__DRIVER_NAME " tcp_length = %d, mss = %d\n", pi->tcp_length, pi->mss));
223 //KdPrint((__DRIVER_NAME " total_length = %d\n", pi->total_length));
224 //KdPrint((__DRIVER_NAME " header_length = %d\n", pi->header_length));
225 //KdPrint((__DRIVER_NAME " header_extra = %d\n", header_extra));
226 if (pi->split_required)
227 {
228 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
229 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
230 //KdPrint((__DRIVER_NAME " new_ip4_length = %d\n", new_ip4_length));
231 //KdPrint((__DRIVER_NAME " this tcp_length = %d\n", tcp_length));
232 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
233 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
234 pi->tcp_seq += tcp_length;
235 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
236 /* part of the packet is already present in the header buffer for lookahead */
237 out_remaining = tcp_length - header_extra;
238 }
239 else
240 {
241 out_remaining = pi->total_length - pi->header_length;
242 }
243 //KdPrint((__DRIVER_NAME " before loop - out_remaining = %d\n", out_remaining));
245 while (out_remaining != 0)
246 {
247 ULONG in_buffer_offset;
248 ULONG in_buffer_length;
249 ULONG out_length;
251 //KdPrint((__DRIVER_NAME " in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_buffer, pi->curr_pb));
252 if (!pi->curr_buffer || !pi->curr_pb)
253 {
254 KdPrint((__DRIVER_NAME " out of buffers for packet\n"));
255 // TODO: free some stuff or we'll leak
256 return NULL;
257 }
258 NdisQueryBufferOffset(pi->curr_buffer, &in_buffer_offset, &in_buffer_length);
259 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
260 NdisCopyBuffer(&status, &out_buffer, xi->rx_buffer_pool, pi->curr_buffer, pi->curr_mdl_offset, out_length);
261 //TODO: check status
262 NdisChainBufferAtBack(packet, out_buffer);
263 ref_pb(xi, pi->curr_pb);
264 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
265 if (pi->curr_mdl_offset == in_buffer_length)
266 {
267 NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
268 pi->curr_pb = pi->curr_pb->next;
269 pi->curr_mdl_offset = 0;
270 }
271 out_remaining -= out_length;
272 }
273 if (pi->split_required)
274 {
275 XenNet_SumIpHeader(header_va, pi->ip4_header_length);
276 }
277 NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
278 if (header_extra > 0)
279 pi->header_length -= header_extra;
280 xi->rx_outstanding++;
281 //FUNCTION_EXIT();
282 return packet;
283 }
285 /*
286 Windows appears to insist that the checksum on received packets is correct, and won't
287 believe us when we lie about it, which happens when the packet is generated on the
288 same bridge in Dom0. Doh!
289 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
290 */
292 static BOOLEAN
293 XenNet_SumPacketData(
294 packet_info_t *pi,
295 PNDIS_PACKET packet,
296 BOOLEAN set_csum
297 )
298 {
299 USHORT i;
300 PUCHAR buffer;
301 PMDL mdl;
302 UINT total_length;
303 UINT data_length;
304 UINT buffer_length;
305 USHORT buffer_offset;
306 ULONG csum;
307 PUSHORT csum_ptr;
308 USHORT remaining;
309 USHORT ip4_length;
310 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
312 //FUNCTION_ENTER();
314 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
315 ASSERT(mdl);
317 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
318 data_length = ip4_length + XN_HDR_SIZE;
320 if ((USHORT)data_length > total_length)
321 {
322 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
323 return FALSE;
324 }
326 switch (pi->ip_proto)
327 {
328 case 6:
329 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
330 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
331 break;
332 case 17:
333 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
334 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
335 break;
336 default:
337 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
338 //FUNCTION_EXIT();
339 return FALSE; // should never happen
340 }
342 if (set_csum)
343 *csum_ptr = 0;
345 csum = 0;
346 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
347 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
348 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
350 remaining = ip4_length - pi->ip4_header_length;
352 csum += remaining;
354 csum_span = FALSE;
355 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
356 while (i < data_length)
357 {
358 /* don't include the checksum field itself in the calculation */
359 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
360 {
361 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
362 i += 2;
363 buffer_offset += 2;
364 continue;
365 }
366 if (csum_span)
367 {
368 /* the other half of the next bit */
369 ASSERT(buffer_offset == 0);
370 csum += (USHORT)buffer[buffer_offset];
371 csum_span = FALSE;
372 i += 1;
373 buffer_offset += 1;
374 }
375 else if (buffer_offset == buffer_length - 1)
376 {
377 /* deal with a buffer ending on an odd byte boundary */
378 csum += (USHORT)buffer[buffer_offset] << 8;
379 csum_span = TRUE;
380 i += 1;
381 buffer_offset += 1;
382 }
383 else
384 {
385 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
386 i += 2;
387 buffer_offset += 2;
388 }
389 if (buffer_offset == buffer_length && i < total_length)
390 {
391 NdisGetNextBuffer(mdl, &mdl);
392 if (mdl == NULL)
393 {
394 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
395 return FALSE; // should never happen
396 }
397 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
398 ASSERT(buffer_length);
399 buffer_offset = 0;
400 }
401 }
403 while (csum & 0xFFFF0000)
404 csum = (csum & 0xFFFF) + (csum >> 16);
406 if (set_csum)
407 {
408 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
409 }
410 else
411 {
412 //FUNCTION_EXIT();
413 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
414 }
415 //FUNCTION_EXIT();
416 return TRUE;
417 }
419 static ULONG
420 XenNet_MakePackets(
421 struct xennet_info *xi,
422 PLIST_ENTRY rx_packet_list
423 )
424 {
425 ULONG packet_count = 0;
426 PNDIS_PACKET packet;
427 PLIST_ENTRY entry;
428 UCHAR psh;
429 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
430 ULONG parse_result;
431 packet_info_t *pi = &xi->rxpi;
432 //PNDIS_BUFFER buffer;
433 shared_buffer_t *page_buf;
435 //FUNCTION_ENTER();
437 parse_result = XenNet_ParsePacketHeader(pi, NULL, 0);
439 if (!XenNet_FilterAcceptPacket(xi, pi))
440 {
441 goto done;
442 }
444 switch (pi->ip_proto)
445 {
446 case 6: // TCP
447 if (pi->split_required)
448 break;
449 // fallthrough
450 case 17: // UDP
451 packet = XenNet_MakePacket(xi);
452 if (packet == NULL)
453 {
454 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
455 xi->stat_rx_no_buffer++;
456 packet_count = 0;
457 goto done;
458 }
459 if (parse_result == PARSE_OK)
460 {
461 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
462 packet, TcpIpChecksumPacketInfo);
463 if (pi->csum_blank || pi->data_validated)
464 {
465 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
466 {
467 if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
468 {
469 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
470 }
471 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
472 {
473 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
474 }
475 if (pi->csum_blank)
476 {
477 XenNet_SumPacketData(pi, packet, TRUE);
478 }
479 }
480 else if (xi->config_csum_rx_check)
481 {
482 if (xi->setting_csum.V4Receive.TcpChecksum && pi->ip_proto == 6)
483 {
484 if (XenNet_SumPacketData(pi, packet, FALSE))
485 {
486 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
487 }
488 else
489 {
490 csum_info->Receive.NdisPacketTcpChecksumFailed = TRUE;
491 }
492 } else if (xi->setting_csum.V4Receive.UdpChecksum && pi->ip_proto == 17)
493 {
494 if (XenNet_SumPacketData(pi, packet, FALSE))
495 {
496 csum_info->Receive.NdisPacketUdpChecksumSucceeded = TRUE;
497 }
498 else
499 {
500 csum_info->Receive.NdisPacketUdpChecksumFailed = TRUE;
501 }
502 }
503 }
504 }
505 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
506 InsertTailList(rx_packet_list, entry);
507 packet_count = 1;
508 goto done;
509 default:
510 packet = XenNet_MakePacket(xi);
511 if (packet == NULL)
512 {
513 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
514 xi->stat_rx_no_buffer++;
515 packet_count = 0;
516 goto done;
517 }
518 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
519 InsertTailList(rx_packet_list, entry);
520 packet_count = 1;
521 goto done;
522 }
524 pi->tcp_remaining = pi->tcp_length;
526 /* we can make certain assumptions here as the following code is only for tcp4 */
527 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
528 while (pi->tcp_remaining)
529 {
530 PUCHAR header_va;
531 PMDL mdl;
532 UINT total_length;
533 UINT buffer_length;
534 packet = XenNet_MakePacket(xi);
535 if (!packet)
536 {
537 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
538 xi->stat_rx_no_buffer++;
539 break; /* we are out of memory - just drop the packets */
540 }
541 if (xi->setting_csum.V4Receive.TcpChecksum)
542 {
543 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
544 packet, TcpIpChecksumPacketInfo);
545 csum_info->Receive.NdisPacketTcpChecksumSucceeded = TRUE;
546 }
547 if (psh)
548 {
549 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
550 if (pi->tcp_remaining)
551 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
552 else
553 header_va[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
554 }
555 XenNet_SumPacketData(pi, packet, TRUE);
556 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[0];
557 InsertTailList(rx_packet_list, entry);
558 packet_count++;
559 }
561 done:
562 page_buf = pi->first_pb;
563 while (page_buf)
564 {
565 shared_buffer_t *next_pb;
567 next_pb = page_buf->next;
568 put_pb_on_freelist(xi, page_buf);
569 page_buf = next_pb;
570 }
571 XenNet_ClearPacketInfo(pi);
572 //FUNCTION_EXIT();
573 return packet_count;
574 }
576 static BOOLEAN
577 XenNet_RxQueueDpcSynchronized(PVOID context)
578 {
579 struct xennet_info *xi = context;
581 KeInsertQueueDpc(&xi->rx_dpc, NULL, NULL);
583 return TRUE;
584 }
586 #define MAXIMUM_PACKETS_PER_INDICATE 32
587 /*
588 We limit the number of packets per interrupt so that acks get a chance
589 under high rx load. The DPC is immediately re-scheduled */
591 #define MAX_PACKETS_PER_INTERRUPT 64
593 // Called at DISPATCH_LEVEL
594 static VOID
595 XenNet_RxBufferCheck(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
596 {
597 struct xennet_info *xi = context;
598 RING_IDX cons, prod;
599 LIST_ENTRY rx_packet_list;
600 PLIST_ENTRY entry;
601 PNDIS_PACKET packets[MAXIMUM_PACKETS_PER_INDICATE];
602 ULONG packet_count = 0;
603 struct netif_rx_response *rxrsp = NULL;
604 struct netif_extra_info *ei;
605 USHORT id;
606 int more_to_do = FALSE;
607 packet_info_t *pi = &xi->rxpi;
608 //NDIS_STATUS status;
609 shared_buffer_t *page_buf;
610 PNDIS_BUFFER buffer;
612 UNREFERENCED_PARAMETER(dpc);
613 UNREFERENCED_PARAMETER(arg1);
614 UNREFERENCED_PARAMETER(arg2);
616 //FUNCTION_ENTER();
618 ASSERT(xi->connected);
620 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
622 if (xi->rx_shutting_down)
623 {
624 /* there is a chance that our Dpc had been queued just before the shutdown... */
625 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
626 return;
627 }
628 InitializeListHead(&rx_packet_list);
630 do {
631 prod = xi->rx.sring->rsp_prod;
632 //KdPrint((__DRIVER_NAME " prod - cons = %d\n", prod - xi->rx.rsp_cons));
633 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
635 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAX_PACKETS_PER_INTERRUPT; cons++)
636 {
637 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
638 ASSERT(xi->rx_ring_pbs[id] != (USHORT)0xFFFF);
639 page_buf = &xi->rx_pbs[xi->rx_ring_pbs[id]];
640 xi->rx_ring_pbs[id] = 0xFFFF;
641 xi->rx_id_free++;
642 //KdPrint((__DRIVER_NAME " got page_buf %p with id %d from ring at id %d\n", page_buf, page_buf->id, id));
643 if (pi->extra_info)
644 {
645 //KdPrint((__DRIVER_NAME " processing extra info\n"));
646 put_pb_on_freelist(xi, page_buf);
647 ei = (struct netif_extra_info *)RING_GET_RESPONSE(&xi->rx, cons);
648 pi->extra_info = (BOOLEAN)!!(ei->flags & XEN_NETIF_EXTRA_FLAG_MORE);
649 switch (ei->type)
650 {
651 case XEN_NETIF_EXTRA_TYPE_GSO:
652 switch (ei->u.gso.type)
653 {
654 case XEN_NETIF_GSO_TYPE_TCPV4:
655 pi->mss = ei->u.gso.size;
656 //KdPrint((__DRIVER_NAME " mss = %d\n", pi->mss));
657 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
658 break;
659 default:
660 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
661 break;
662 }
663 break;
664 default:
665 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
666 break;
667 }
668 }
669 else
670 {
671 rxrsp = RING_GET_RESPONSE(&xi->rx, cons);
672 if (rxrsp->status <= 0
673 || rxrsp->offset + rxrsp->status > PAGE_SIZE)
674 {
675 KdPrint((__DRIVER_NAME " Error: rxrsp offset %d, size %d\n",
676 rxrsp->offset, rxrsp->status));
677 ASSERT(!pi->extra_info);
678 put_pb_on_freelist(xi, page_buf);
679 continue;
680 }
681 ASSERT(!rxrsp->offset);
682 ASSERT(rxrsp->id == id);
683 if (!pi->more_frags) // handling the packet's 1st buffer
684 {
685 if (rxrsp->flags & NETRXF_csum_blank)
686 pi->csum_blank = TRUE;
687 if (rxrsp->flags & NETRXF_data_validated)
688 pi->data_validated = TRUE;
689 }
690 //NdisAllocateBuffer(&status, &buffer, xi->rx_buffer_pool, (PUCHAR)page_buf->virtual + rxrsp->offset, rxrsp->status);
691 //KdPrint((__DRIVER_NAME " buffer = %p, offset = %d, len = %d\n", buffer, rxrsp->offset, rxrsp->status));
692 //ASSERT(status == NDIS_STATUS_SUCCESS); // lazy
693 buffer = page_buf->buffer;
694 NdisAdjustBufferLength(buffer, rxrsp->status);
695 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
696 if (pi->first_pb)
697 {
698 //KdPrint((__DRIVER_NAME " additional buffer\n"));
699 pi->curr_pb->next = page_buf;
700 pi->curr_pb = page_buf;
701 NDIS_BUFFER_LINKAGE(pi->curr_buffer) = buffer;
702 pi->curr_buffer = buffer;
703 }
704 else
705 {
706 pi->first_pb = page_buf;
707 pi->curr_pb = page_buf;
708 pi->first_buffer = buffer;
709 pi->curr_buffer = buffer;
710 }
711 pi->mdl_count++;
712 pi->extra_info = (BOOLEAN)!!(rxrsp->flags & NETRXF_extra_info);
713 pi->more_frags = (BOOLEAN)!!(rxrsp->flags & NETRXF_more_data);
714 pi->total_length = pi->total_length + rxrsp->status;
715 }
717 /* Packet done, add it to the list */
718 if (!pi->more_frags && !pi->extra_info)
719 {
720 pi->curr_pb = pi->first_pb;
721 pi->curr_buffer = pi->first_buffer;
722 packet_count += XenNet_MakePackets(xi, &rx_packet_list);
723 }
724 }
725 xi->rx.rsp_cons = cons;
727 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
728 break;
730 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
731 if (!more_to_do)
732 {
733 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
734 mb();
735 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
736 }
737 } while (more_to_do);
739 if (pi->more_frags || pi->extra_info)
740 KdPrint((__DRIVER_NAME " Partial receive (more_frags = %d, extra_info = %d, total_length = %d, mdl_count = %d)\n", pi->more_frags, pi->extra_info, pi->total_length, pi->mdl_count));
742 /* Give netback more buffers */
743 XenNet_FillRing(xi);
745 if (packet_count >= MAX_PACKETS_PER_INTERRUPT)
746 {
747 /* fire again immediately */
748 xi->vectors.EvtChn_Sync(xi->vectors.context, XenNet_RxQueueDpcSynchronized, xi);
749 }
751 //KdPrint((__DRIVER_NAME " packet_count = %d, page_count = %d, avg_page_count = %d, event = %d\n", packet_count, page_count, xi->avg_page_count / 128, event));
752 xi->stat_rx_ok += packet_count;
754 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
756 entry = RemoveHeadList(&rx_packet_list);
757 packet_count = 0;
758 while (entry != &rx_packet_list)
759 {
760 PNDIS_PACKET packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[0]);
761 packets[packet_count++] = packet;
762 entry = RemoveHeadList(&rx_packet_list);
763 if (packet_count == MAXIMUM_PACKETS_PER_INDICATE || entry == &rx_packet_list)
764 {
765 //KdPrint((__DRIVER_NAME " Indicating\n"));
766 NdisMIndicateReceivePacket(xi->adapter_handle, packets, packet_count);
767 packet_count = 0;
768 }
769 }
770 //FUNCTION_EXIT();
771 }
773 /* called at DISPATCH_LEVEL */
774 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
775 VOID DDKAPI
776 XenNet_ReturnPacket(
777 IN NDIS_HANDLE MiniportAdapterContext,
778 IN PNDIS_PACKET Packet
779 )
780 {
781 struct xennet_info *xi = MiniportAdapterContext;
782 PNDIS_BUFFER buffer;
783 shared_buffer_t *page_buf = *(shared_buffer_t **)&Packet->MiniportReservedEx[sizeof(LIST_ENTRY)];
785 //FUNCTION_ENTER();
787 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
789 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
791 NdisUnchainBufferAtFront(Packet, &buffer);
792 while (buffer)
793 {
794 shared_buffer_t *next_buf = page_buf->next;
795 if (!page_buf->virtual)
796 {
797 /* this isn't actually a share_buffer, it is some memory allocated for the header - just free it */
798 PUCHAR va;
799 UINT len;
800 NdisQueryBufferSafe(buffer, &va, &len, NormalPagePriority);
801 NdisFreeToNPagedLookasideList(&xi->rx_lookaside_list, va - sizeof(shared_buffer_t));
802 NdisFreeBuffer(buffer);
803 }
804 else
805 {
806 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
807 if (buffer != page_buf->buffer)
808 NdisFreeBuffer(buffer);
809 put_pb_on_freelist(xi, page_buf);
810 }
811 NdisUnchainBufferAtFront(Packet, &buffer);
812 page_buf = next_buf;
813 }
815 put_packet_on_freelist(xi, Packet);
816 xi->rx_outstanding--;
818 if (!xi->rx_outstanding && xi->rx_shutting_down)
819 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
821 XenNet_FillRing(xi);
823 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
825 //FUNCTION_EXIT();
826 }
828 /*
829 Free all Rx buffers (on halt, for example)
830 The ring must be stopped at this point.
831 */
833 static VOID
834 XenNet_PurgeRing(struct xennet_info *xi)
835 {
836 int i;
837 for (i = 0; i < NET_RX_RING_SIZE; i++)
838 {
839 if (xi->rx_ring_pbs[i] != 0xFFFF)
840 {
841 put_pb_on_freelist(xi, &xi->rx_pbs[xi->rx_ring_pbs[i]]);
842 xi->rx_ring_pbs[i] = 0xFFFF;
843 }
844 }
845 }
847 static VOID
848 XenNet_BufferFree(struct xennet_info *xi)
849 {
850 shared_buffer_t *pb;
852 XenNet_PurgeRing(xi);
854 while ((pb = get_pb_from_freelist(xi)) != NULL)
855 {
856 NdisFreeBuffer(pb->buffer);
857 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
858 pb->gref, FALSE);
859 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
860 }
861 }
863 VOID
864 XenNet_RxResumeStart(xennet_info_t *xi)
865 {
866 KIRQL old_irql;
868 FUNCTION_ENTER();
870 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
871 XenNet_PurgeRing(xi);
872 KeReleaseSpinLock(&xi->rx_lock, old_irql);
874 FUNCTION_EXIT();
875 }
877 VOID
878 XenNet_BufferAlloc(xennet_info_t *xi)
879 {
880 NDIS_STATUS status;
881 int i;
883 xi->rx_id_free = NET_RX_RING_SIZE;
884 xi->rx_outstanding = 0;
886 for (i = 0; i < NET_RX_RING_SIZE; i++)
887 {
888 xi->rx_ring_pbs[i] = 0xFFFF;
889 }
891 for (i = 0; i < RX_PAGE_BUFFERS; i++)
892 {
893 xi->rx_pbs[i].id = (USHORT)i;
894 status = NdisAllocateMemoryWithTag(&xi->rx_pbs[i].virtual, PAGE_SIZE, XENNET_POOL_TAG);
895 if (status != STATUS_SUCCESS)
896 {
897 break;
898 }
899 xi->rx_pbs[i].gref = (grant_ref_t)xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
900 (ULONG)(MmGetPhysicalAddress(xi->rx_pbs[i].virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF);
901 if (xi->rx_pbs[i].gref == INVALID_GRANT_REF)
902 {
903 NdisFreeMemory(xi->rx_pbs[i].virtual, PAGE_SIZE, 0);
904 break;
905 }
906 xi->rx_pbs[i].offset = (ULONG_PTR)xi->rx_pbs[i].virtual & (PAGE_SIZE - 1);
907 NdisAllocateBuffer(&status, &xi->rx_pbs[i].buffer, xi->rx_buffer_pool, (PUCHAR)xi->rx_pbs[i].virtual, PAGE_SIZE);
908 if (status != STATUS_SUCCESS)
909 {
910 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
911 xi->rx_pbs[i].gref, FALSE);
912 NdisFreeMemory(xi->rx_pbs[i].virtual, PAGE_SIZE, 0);
913 break;
914 }
915 xi->rx_pbs[i].ref_count = 1; /* when we put it back it will go to zero */
916 put_pb_on_freelist(xi, &xi->rx_pbs[i]);
917 }
918 if (i == 0)
919 KdPrint((__DRIVER_NAME " Unable to allocate any SharedMemory buffers\n"));
920 }
923 VOID
924 XenNet_RxResumeEnd(xennet_info_t *xi)
925 {
926 KIRQL old_irql;
928 FUNCTION_ENTER();
930 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
931 //XenNet_BufferAlloc(xi);
932 XenNet_FillRing(xi);
933 KeReleaseSpinLock(&xi->rx_lock, old_irql);
935 FUNCTION_EXIT();
936 }
938 BOOLEAN
939 XenNet_RxInit(xennet_info_t *xi)
940 {
941 NDIS_STATUS status;
943 FUNCTION_ENTER();
945 xi->rx_shutting_down = FALSE;
946 KeInitializeSpinLock(&xi->rx_lock);
947 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
948 KeInitializeTimer(&xi->rx_timer);
949 KeInitializeDpc(&xi->rx_dpc, XenNet_RxBufferCheck, xi);
950 KeSetTargetProcessorDpc(&xi->rx_dpc, 0);
951 //KeSetImportanceDpc(&xi->rx_dpc, HighImportance);
952 //KeInitializeDpc(&xi->rx_timer_dpc, XenNet_RxTimerDpc, xi);
954 XenNet_BufferAlloc(xi);
956 NdisAllocatePacketPool(&status, &xi->rx_packet_pool, NET_RX_RING_SIZE * 4,
957 PROTOCOL_RESERVED_SIZE_IN_PACKET);
958 if (status != NDIS_STATUS_SUCCESS)
959 {
960 KdPrint(("NdisAllocatePacketPool failed with 0x%x\n", status));
961 return FALSE;
962 }
964 NdisInitializeNPagedLookasideList(&xi->rx_lookaside_list, NULL, NULL, 0,
965 MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH + sizeof(shared_buffer_t), XENNET_POOL_TAG, 0);
967 XenNet_FillRing(xi);
969 FUNCTION_EXIT();
971 return TRUE;
972 }
974 BOOLEAN
975 XenNet_RxShutdown(xennet_info_t *xi)
976 {
977 KIRQL old_irql;
979 FUNCTION_ENTER();
981 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
982 xi->rx_shutting_down = TRUE;
983 KeReleaseSpinLock(&xi->rx_lock, old_irql);
985 if (xi->config_rx_interrupt_moderation)
986 {
987 KeCancelTimer(&xi->rx_timer);
988 }
990 KeRemoveQueueDpc(&xi->rx_dpc);
991 KeFlushQueuedDpcs();
993 while (xi->rx_outstanding)
994 {
995 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
996 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
997 }
999 //KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1001 XenNet_BufferFree(xi);
1003 packet_freelist_dispose(xi);
1005 NdisFreePacketPool(xi->rx_packet_pool);
1007 NdisDeleteNPagedLookasideList(&xi->rx_lookaside_list);
1009 //KeReleaseSpinLock(&xi->rx_lock, old_irql);
1011 FUNCTION_EXIT();
1013 return TRUE;