win-pvdrivers

view xennet/xennet6_rx.c @ 951:013b63248e5d

Fix FreeMemory size parameter
author James Harper <james.harper@bendigoit.com.au>
date Thu Nov 10 19:27:28 2011 +1100 (2011-11-10)
parents 916ea40186fc
children 278b479f3f7d
line source
1 /*
2 PV Net Driver for Windows Xen HVM Domains
3 Copyright (C) 2007 James Harper
4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
6 This program is free software; you can redistribute it and/or
7 modify it under the terms of the GNU General Public License
8 as published by the Free Software Foundation; either version 2
9 of the License, or (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 */
21 #include "xennet6.h"
23 static __inline shared_buffer_t *
24 get_pb_from_freelist(struct xennet_info *xi)
25 {
26 shared_buffer_t *pb;
27 PVOID ptr_ref;
29 if (stack_pop(xi->rx_pb_stack, &ptr_ref))
30 {
31 pb = ptr_ref;
32 pb->ref_count = 1;
33 InterlockedDecrement(&xi->rx_pb_free);
34 return pb;
35 }
37 /* don't allocate a new one if we are shutting down */
38 if (xi->shutting_down)
39 return NULL;
41 pb = NdisAllocateMemoryWithTagPriority(xi->adapter_handle, sizeof(shared_buffer_t), XENNET_POOL_TAG, LowPoolPriority);
42 if (!pb)
43 return NULL;
44 pb->virtual = NdisAllocateMemoryWithTagPriority(xi->adapter_handle, PAGE_SIZE, XENNET_POOL_TAG, LowPoolPriority);
45 if (!pb->virtual)
46 {
47 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
48 return NULL;
49 }
50 pb->mdl = IoAllocateMdl(pb->virtual, PAGE_SIZE, FALSE, FALSE, NULL);
51 if (!pb->mdl)
52 {
53 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
54 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
55 return NULL;
56 }
57 pb->gref = (grant_ref_t)xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
58 (ULONG)(MmGetPhysicalAddress(pb->virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF, (ULONG)'XNRX');
59 if (pb->gref == INVALID_GRANT_REF)
60 {
61 IoFreeMdl(pb->mdl);
62 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
63 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
64 return NULL;
65 }
66 MmBuildMdlForNonPagedPool(pb->mdl);
67 pb->ref_count = 1;
68 return pb;
69 }
71 static __inline VOID
72 ref_pb(struct xennet_info *xi, shared_buffer_t *pb)
73 {
74 UNREFERENCED_PARAMETER(xi);
75 InterlockedIncrement(&pb->ref_count);
76 }
78 static __inline VOID
79 put_pb_on_freelist(struct xennet_info *xi, shared_buffer_t *pb)
80 {
81 if (InterlockedDecrement(&pb->ref_count) == 0)
82 {
83 //NdisAdjustBufferLength(pb->buffer, PAGE_SIZE);
84 //NDIS_BUFFER_LINKAGE(pb->buffer) = NULL;
85 if (xi->rx_pb_free > RX_MAX_PB_FREELIST)
86 {
87 IoFreeMdl(pb->mdl);
88 NdisFreeMemory(pb->virtual, PAGE_SIZE, 0);
89 NdisFreeMemory(pb, sizeof(shared_buffer_t), 0);
90 return;
91 }
92 pb->mdl->ByteCount = PAGE_SIZE;
93 pb->mdl->Next = NULL;
94 pb->next = NULL;
95 stack_push(xi->rx_pb_stack, pb);
96 InterlockedIncrement(&xi->rx_pb_free);
97 }
98 }
100 static __inline shared_buffer_t *
101 get_hb_from_freelist(struct xennet_info *xi)
102 {
103 shared_buffer_t *hb;
104 PVOID ptr_ref;
106 if (stack_pop(xi->rx_hb_stack, &ptr_ref))
107 {
108 hb = ptr_ref;
109 InterlockedDecrement(&xi->rx_hb_free);
110 return hb;
111 }
113 /* don't allocate a new one if we are shutting down */
114 if (xi->shutting_down)
115 return NULL;
117 hb = NdisAllocateMemoryWithTagPriority(xi->adapter_handle, sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, XENNET_POOL_TAG, LowPoolPriority);
118 if (!hb)
119 return NULL;
120 NdisZeroMemory(hb, sizeof(shared_buffer_t));
121 hb->mdl = IoAllocateMdl(hb + 1, MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, FALSE, FALSE, NULL);
122 if (!hb->mdl)
123 {
124 NdisFreeMemory(hb, sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, 0);
125 return NULL;
126 }
127 MmBuildMdlForNonPagedPool(hb->mdl);
128 return hb;
129 }
131 static __inline VOID
132 put_hb_on_freelist(struct xennet_info *xi, shared_buffer_t *hb)
133 {
134 ASSERT(xi);
135 hb->mdl->ByteCount = sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH;
136 hb->mdl->Next = NULL;
137 hb->next = NULL;
138 stack_push(xi->rx_hb_stack, hb);
139 InterlockedIncrement(&xi->rx_hb_free);
140 }
142 // Called at DISPATCH_LEVEL with rx lock held
143 static NDIS_STATUS
144 XenNet_FillRing(struct xennet_info *xi)
145 {
146 unsigned short id;
147 shared_buffer_t *page_buf;
148 ULONG i, notify;
149 ULONG batch_target;
150 RING_IDX req_prod = xi->rx.req_prod_pvt;
151 netif_rx_request_t *req;
153 //FUNCTION_ENTER();
155 batch_target = xi->rx_target - (req_prod - xi->rx.rsp_cons);
157 if (batch_target < (xi->rx_target >> 2))
158 {
159 //FUNCTION_EXIT();
160 return NDIS_STATUS_SUCCESS; /* only refill if we are less than 3/4 full already */
161 }
163 for (i = 0; i < batch_target; i++)
164 {
165 page_buf = get_pb_from_freelist(xi);
166 if (!page_buf)
167 {
168 KdPrint((__DRIVER_NAME " Added %d out of %d buffers to rx ring (no free pages)\n", i, batch_target));
169 break;
170 }
171 xi->rx_id_free--;
173 /* Give to netback */
174 id = (USHORT)((req_prod + i) & (NET_RX_RING_SIZE - 1));
175 ASSERT(xi->rx_ring_pbs[id] == NULL);
176 xi->rx_ring_pbs[id] = page_buf;
177 req = RING_GET_REQUEST(&xi->rx, req_prod + i);
178 req->id = id;
179 req->gref = page_buf->gref;
180 ASSERT(req->gref != INVALID_GRANT_REF);
181 }
182 KeMemoryBarrier();
183 xi->rx.req_prod_pvt = req_prod + i;
184 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->rx, notify);
185 if (notify)
186 {
187 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
188 }
190 //FUNCTION_EXIT();
192 return NDIS_STATUS_SUCCESS;
193 }
195 typedef struct {
196 PNET_BUFFER_LIST first_nbl;
197 PNET_BUFFER_LIST last_nbl;
198 ULONG packet_count;
199 ULONG nbl_count;
200 } rx_context_t;
202 static BOOLEAN
203 XenNet_MakePacket(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi)
204 {
205 PNET_BUFFER_LIST nbl;
206 PNET_BUFFER nb;
207 PMDL mdl_head, mdl_tail, curr_mdl;
208 PUCHAR header_va;
209 ULONG out_remaining;
210 ULONG header_extra;
211 shared_buffer_t *header_buf;
212 NDIS_TCP_IP_CHECKSUM_NET_BUFFER_LIST_INFO csum_info;
214 //FUNCTION_ENTER();
216 nbl = NdisAllocateNetBufferList(xi->rx_nbl_pool, 0, 0);
217 if (!nbl)
218 {
219 /* buffers will be freed in MakePackets */
220 KdPrint((__DRIVER_NAME " No free nbl's\n"));
221 //FUNCTION_EXIT();
222 return FALSE;
223 }
225 nb = NdisAllocateNetBuffer(xi->rx_nb_pool, NULL, 0, 0);
226 if (!nb)
227 {
228 KdPrint((__DRIVER_NAME " No free nb's\n"));
229 NdisFreeNetBufferList(nbl);
230 //FUNCTION_EXIT();
231 return FALSE;
232 }
234 header_buf = get_hb_from_freelist(xi);
235 if (!header_buf)
236 {
237 KdPrint((__DRIVER_NAME " No free header buffers\n"));
238 NdisFreeNetBufferList(nbl);
239 NdisFreeNetBuffer(nb);
240 //FUNCTION_EXIT();
241 return FALSE;
242 }
243 header_va = (PUCHAR)(header_buf + 1);
244 NdisMoveMemory(header_va, pi->header, pi->header_length);
245 //KdPrint((__DRIVER_NAME " header_length = %d, current_lookahead = %d\n", pi->header_length, xi->current_lookahead));
246 //KdPrint((__DRIVER_NAME " ip4_header_length = %d\n", pi->ip4_header_length));
247 //KdPrint((__DRIVER_NAME " tcp_header_length = %d\n", pi->tcp_header_length));
248 /* make sure we satisfy the lookahead requirement */
249 if (pi->split_required)
250 {
251 /* for split packets we need to make sure the 'header' is no bigger than header+mss bytes */
252 XenNet_BuildHeader(pi, header_va, min((ULONG)MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length + pi->mss, MAX_ETH_HEADER_LENGTH + max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead)));
253 }
254 else
255 {
256 XenNet_BuildHeader(pi, header_va, max(MIN_LOOKAHEAD_LENGTH, xi->current_lookahead) + MAX_ETH_HEADER_LENGTH);
257 }
258 header_extra = pi->header_length - (MAX_ETH_HEADER_LENGTH + pi->ip4_header_length + pi->tcp_header_length);
259 ASSERT(pi->header_length <= MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH);
260 header_buf->mdl->ByteCount = pi->header_length;
261 mdl_head = mdl_tail = curr_mdl = header_buf->mdl;
262 NB_HEADER_BUF(nb) = header_buf;
263 header_buf->next = pi->curr_pb;
264 NET_BUFFER_FIRST_MDL(nb) = mdl_head;
265 NET_BUFFER_CURRENT_MDL(nb) = mdl_head;
266 NET_BUFFER_CURRENT_MDL_OFFSET(nb) = 0;
267 NET_BUFFER_DATA_OFFSET(nb) = 0;
268 NET_BUFFER_DATA_LENGTH(nb) = pi->header_length;
270 // TODO: if there are only a few bytes left on the first buffer then add them to the header buffer too... maybe
272 if (pi->split_required)
273 {
274 ULONG tcp_length;
275 USHORT new_ip4_length;
276 tcp_length = (USHORT)min(pi->mss, pi->tcp_remaining);
277 new_ip4_length = (USHORT)(pi->ip4_header_length + pi->tcp_header_length + tcp_length);
278 //KdPrint((__DRIVER_NAME " new_ip4_length = %d\n", new_ip4_length));
279 //KdPrint((__DRIVER_NAME " this tcp_length = %d\n", tcp_length));
280 SET_NET_USHORT(&header_va[XN_HDR_SIZE + 2], new_ip4_length);
281 SET_NET_ULONG(&header_va[XN_HDR_SIZE + pi->ip4_header_length + 4], pi->tcp_seq);
282 pi->tcp_seq += tcp_length;
283 pi->tcp_remaining = (USHORT)(pi->tcp_remaining - tcp_length);
284 /* part of the packet is already present in the header buffer for lookahead */
285 out_remaining = tcp_length - header_extra;
286 ASSERT((LONG)out_remaining >= 0);
287 }
288 else
289 {
290 out_remaining = pi->total_length - pi->header_length;
291 ASSERT((LONG)out_remaining >= 0);
292 }
293 //KdPrint((__DRIVER_NAME " before loop - out_remaining = %d\n", out_remaining));
295 while (out_remaining != 0)
296 {
297 //ULONG in_buffer_offset;
298 ULONG in_buffer_length;
299 ULONG out_length;
301 //KdPrint((__DRIVER_NAME " in loop - out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb));
302 if (!pi->curr_mdl || !pi->curr_pb)
303 {
304 KdPrint((__DRIVER_NAME " out of buffers for packet\n"));
305 //KdPrint((__DRIVER_NAME " out_remaining = %d, curr_buffer = %p, curr_pb = %p\n", out_remaining, pi->curr_mdl, pi->curr_pb));
306 // TODO: free some stuff or we'll leak
307 /* unchain buffers then free packet */
308 //FUNCTION_EXIT();
309 return FALSE;
310 }
312 in_buffer_length = MmGetMdlByteCount(pi->curr_mdl);
313 out_length = min(out_remaining, in_buffer_length - pi->curr_mdl_offset);
314 curr_mdl = IoAllocateMdl((PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length, FALSE, FALSE, NULL);
315 ASSERT(curr_mdl);
316 IoBuildPartialMdl(pi->curr_mdl, curr_mdl, (PUCHAR)MmGetMdlVirtualAddress(pi->curr_mdl) + pi->curr_mdl_offset, out_length);
317 mdl_tail->Next = curr_mdl;
318 mdl_tail = curr_mdl;
319 curr_mdl->Next = NULL; /* I think this might be redundant */
320 NET_BUFFER_DATA_LENGTH(nb) += out_length;
321 ref_pb(xi, pi->curr_pb);
322 pi->curr_mdl_offset = (USHORT)(pi->curr_mdl_offset + out_length);
323 if (pi->curr_mdl_offset == in_buffer_length)
324 {
325 pi->curr_mdl = pi->curr_mdl->Next;
326 pi->curr_pb = pi->curr_pb->next;
327 pi->curr_mdl_offset = 0;
328 }
329 out_remaining -= out_length;
330 }
331 if (pi->split_required)
332 {
333 // TODO: only if Ip checksum is disabled...
334 //XenNet_SumIpHeader(header_va, pi->ip4_header_length);
335 }
336 if (header_extra > 0)
337 pi->header_length -= header_extra;
338 //ASSERT(*(shared_buffer_t **)&packet->MiniportReservedEx[0]);
340 rc->packet_count++;
341 NET_BUFFER_LIST_FIRST_NB(nbl) = nb;
342 //NET_BUFFER_NEXT_NB(nb) = NULL; /*is this already done for me? */
344 if (pi->parse_result == PARSE_OK)
345 {
346 BOOLEAN checksum_offload = FALSE;
347 csum_info.Value = 0;
348 if (pi->csum_blank || pi->data_validated || pi->mss)
349 {
350 if (pi->ip_proto == 6) // && xi->setting_csum.V4Receive.TcpChecksum)
351 {
352 // if (!pi->tcp_has_options || xi->setting_csum.V4Receive.TcpOptionsSupported)
353 // {
354 csum_info.Receive.IpChecksumSucceeded = TRUE;
355 csum_info.Receive.TcpChecksumSucceeded = TRUE;
356 checksum_offload = TRUE;
357 // }
358 }
359 else if (pi->ip_proto == 17) // &&xi->setting_csum.V4Receive.UdpChecksum)
360 {
361 csum_info.Receive.IpChecksumSucceeded = TRUE;
362 csum_info.Receive.UdpChecksumSucceeded = TRUE;
363 checksum_offload = TRUE;
364 }
365 }
366 NET_BUFFER_LIST_INFO(nbl, TcpIpChecksumNetBufferListInfo) = csum_info.Value;
367 }
369 if (!rc->first_nbl)
370 {
371 rc->first_nbl = nbl;
372 }
373 else
374 {
375 NET_BUFFER_LIST_NEXT_NBL(rc->last_nbl) = nbl;
376 }
377 rc->last_nbl = nbl;
378 NET_BUFFER_LIST_NEXT_NBL(nbl) = NULL;
379 rc->nbl_count++;
380 InterlockedIncrement(&xi->rx_outstanding);
381 if (pi->is_multicast)
382 {
383 /* multicast */
384 xi->stats.ifHCInMulticastPkts++;
385 xi->stats.ifHCInMulticastOctets += NET_BUFFER_DATA_LENGTH(nb);
386 }
387 else if (pi->is_broadcast)
388 {
389 /* broadcast */
390 xi->stats.ifHCInBroadcastPkts++;
391 xi->stats.ifHCInBroadcastOctets += NET_BUFFER_DATA_LENGTH(nb);
392 }
393 else
394 {
395 /* unicast */
396 xi->stats.ifHCInUcastPkts++;
397 xi->stats.ifHCInUcastOctets += NET_BUFFER_DATA_LENGTH(nb);
398 }
399 //FUNCTION_EXIT();
400 return TRUE;
401 }
403 #if 0
404 /*
405 Windows appears to insist that the checksum on received packets is correct, and won't
406 believe us when we lie about it, which happens when the packet is generated on the
407 same bridge in Dom0. Doh!
408 This is only for TCP and UDP packets. IP checksums appear to be correct anyways.
409 */
411 static BOOLEAN
412 XenNet_SumPacketData(
413 packet_info_t *pi,
414 PNDIS_PACKET packet,
415 BOOLEAN set_csum
416 )
417 {
418 USHORT i;
419 PUCHAR buffer;
420 PMDL mdl;
421 UINT total_length;
422 UINT data_length;
423 UINT buffer_length;
424 USHORT buffer_offset;
425 ULONG csum;
426 PUSHORT csum_ptr;
427 USHORT remaining;
428 USHORT ip4_length;
429 BOOLEAN csum_span = TRUE; /* when the USHORT to be checksummed spans a buffer */
431 //FUNCTION_ENTER();
433 NdisGetFirstBufferFromPacketSafe(packet, &mdl, &buffer, &buffer_length, &total_length, NormalPagePriority);
434 ASSERT(mdl);
436 ip4_length = GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 2]);
437 data_length = ip4_length + XN_HDR_SIZE;
439 if ((USHORT)data_length > total_length)
440 {
441 KdPrint((__DRIVER_NAME " Size Mismatch %d (ip4_length + XN_HDR_SIZE) != %d (total_length)\n", ip4_length + XN_HDR_SIZE, total_length));
442 return FALSE;
443 }
445 switch (pi->ip_proto)
446 {
447 case 6:
448 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 17));
449 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 16];
450 break;
451 case 17:
452 ASSERT(buffer_length >= (USHORT)(XN_HDR_SIZE + pi->ip4_header_length + 7));
453 csum_ptr = (USHORT *)&buffer[XN_HDR_SIZE + pi->ip4_header_length + 6];
454 break;
455 default:
456 KdPrint((__DRIVER_NAME " Don't know how to calc sum for IP Proto %d\n", pi->ip_proto));
457 //FUNCTION_EXIT();
458 return FALSE; // should never happen
459 }
461 if (set_csum)
462 *csum_ptr = 0;
464 csum = 0;
465 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 12]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 14]); // src
466 csum += GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 16]) + GET_NET_PUSHORT(&buffer[XN_HDR_SIZE + 18]); // dst
467 csum += ((USHORT)buffer[XN_HDR_SIZE + 9]);
469 remaining = ip4_length - pi->ip4_header_length;
471 csum += remaining;
473 csum_span = FALSE;
474 buffer_offset = i = XN_HDR_SIZE + pi->ip4_header_length;
475 while (i < data_length)
476 {
477 /* don't include the checksum field itself in the calculation */
478 if ((pi->ip_proto == 6 && i == XN_HDR_SIZE + pi->ip4_header_length + 16) || (pi->ip_proto == 17 && i == XN_HDR_SIZE + pi->ip4_header_length + 6))
479 {
480 /* we know that this always happens in the header buffer so we are guaranteed the full two bytes */
481 i += 2;
482 buffer_offset += 2;
483 continue;
484 }
485 if (csum_span)
486 {
487 /* the other half of the next bit */
488 ASSERT(buffer_offset == 0);
489 csum += (USHORT)buffer[buffer_offset];
490 csum_span = FALSE;
491 i += 1;
492 buffer_offset += 1;
493 }
494 else if (buffer_offset == buffer_length - 1)
495 {
496 /* deal with a buffer ending on an odd byte boundary */
497 csum += (USHORT)buffer[buffer_offset] << 8;
498 csum_span = TRUE;
499 i += 1;
500 buffer_offset += 1;
501 }
502 else
503 {
504 csum += GET_NET_PUSHORT(&buffer[buffer_offset]);
505 i += 2;
506 buffer_offset += 2;
507 }
508 if (buffer_offset == buffer_length && i < total_length)
509 {
510 NdisGetNextBuffer(mdl, &mdl);
511 if (mdl == NULL)
512 {
513 KdPrint((__DRIVER_NAME " Ran out of buffers\n"));
514 return FALSE; // should never happen
515 }
516 NdisQueryBufferSafe(mdl, &buffer, &buffer_length, NormalPagePriority);
517 ASSERT(buffer_length);
518 buffer_offset = 0;
519 }
520 }
522 while (csum & 0xFFFF0000)
523 csum = (csum & 0xFFFF) + (csum >> 16);
525 if (set_csum)
526 {
527 *csum_ptr = (USHORT)~GET_NET_USHORT((USHORT)csum);
528 }
529 else
530 {
531 //FUNCTION_EXIT();
532 return (BOOLEAN)(*csum_ptr == (USHORT)~GET_NET_USHORT((USHORT)csum));
533 }
534 //FUNCTION_EXIT();
535 return TRUE;
536 }
537 #endif
539 static VOID
540 XenNet_MakePackets(struct xennet_info *xi, rx_context_t *rc, packet_info_t *pi)
541 {
542 UCHAR psh;
543 //PNDIS_BUFFER buffer;
544 shared_buffer_t *page_buf;
546 //FUNCTION_ENTER();
548 XenNet_ParsePacketHeader(pi, NULL, 0);
549 //pi->split_required = FALSE;
551 if (!XenNet_FilterAcceptPacket(xi, pi))
552 {
553 goto done;
554 }
556 if (pi->split_required)
557 {
558 switch (xi->current_gso_rx_split_type)
559 {
560 case RX_LSO_SPLIT_HALF:
561 pi->mss = (pi->tcp_length + 1) / 2;
562 break;
563 case RX_LSO_SPLIT_NONE:
564 pi->mss = 65535;
565 break;
566 }
567 }
569 switch (pi->ip_proto)
570 {
571 case 6: // TCP
572 if (pi->split_required)
573 break;
574 /* fall through */
575 case 17: // UDP
576 if (!XenNet_MakePacket(xi, rc, pi))
577 {
578 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
579 xi->stats.ifInDiscards++;
580 goto done;
581 }
582 goto done;
583 default:
584 if (!XenNet_MakePacket(xi, rc, pi))
585 {
586 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
587 xi->stats.ifInDiscards++;
588 goto done;
589 }
590 goto done;
591 }
593 /* this is the split_required code */
594 pi->tcp_remaining = pi->tcp_length;
596 /* we can make certain assumptions here as the following code is only for tcp4 */
597 psh = pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] & 8;
598 while (pi->tcp_remaining)
599 {
600 if (!XenNet_MakePacket(xi, rc, pi))
601 {
602 KdPrint((__DRIVER_NAME " Ran out of packets\n"));
603 xi->stats.ifInDiscards++;
604 break; /* we are out of memory - just drop the packets */
605 }
606 if (psh)
607 {
608 //NdisGetFirstBufferFromPacketSafe(packet, &mdl, &header_va, &buffer_length, &total_length, NormalPagePriority);
609 if (pi->tcp_remaining)
610 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] &= ~8;
611 else
612 pi->header[XN_HDR_SIZE + pi->ip4_header_length + 13] |= 8;
613 }
614 //XenNet_SumPacketData(pi, packet, TRUE);
615 //entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
616 //InsertTailList(rx_packet_list, entry);
617 }
618 done:
619 page_buf = pi->first_pb;
620 while (page_buf)
621 {
622 shared_buffer_t *next_pb = page_buf->next;
623 put_pb_on_freelist(xi, page_buf); /* this doesn't actually free the page_puf if there are outstanding references */
624 page_buf = next_pb;
625 }
626 XenNet_ClearPacketInfo(pi);
627 //FUNCTION_EXIT();
628 return;
629 }
631 /* called at <= DISPATCH_LEVEL */
632 /* it's okay for return packet to be called while resume_state != RUNNING as the packet will simply be added back to the freelist, the grants will be fixed later */
633 VOID
634 XenNet_ReturnNetBufferLists(NDIS_HANDLE adapter_context, PNET_BUFFER_LIST curr_nbl, ULONG return_flags)
635 {
636 struct xennet_info *xi = adapter_context;
637 UNREFERENCED_PARAMETER(return_flags);
639 //FUNCTION_ENTER();
641 //KdPrint((__DRIVER_NAME " page_buf = %p\n", page_buf));
643 ASSERT(xi);
644 while (curr_nbl)
645 {
646 PNET_BUFFER_LIST next_nbl;
647 PNET_BUFFER curr_nb;
649 next_nbl = NET_BUFFER_LIST_NEXT_NBL(curr_nbl);
650 curr_nb = NET_BUFFER_LIST_FIRST_NB(curr_nbl);
651 while (curr_nb)
652 {
653 PNET_BUFFER next_nb;
654 PMDL curr_mdl;
655 shared_buffer_t *page_buf;
657 next_nb = NET_BUFFER_NEXT_NB(curr_nb);
658 curr_mdl = NET_BUFFER_FIRST_MDL(curr_nb);
659 page_buf = NB_HEADER_BUF(curr_nb);
660 while (curr_mdl)
661 {
662 shared_buffer_t *next_buf;
663 PMDL next_mdl;
665 ASSERT(page_buf); /* make sure that there is a pb to match this mdl */
666 next_mdl = curr_mdl->Next;
667 next_buf = page_buf->next;
668 if (!page_buf->virtual)
669 {
670 /* this is a hb not a pb because virtual is NULL (virtual is just the memory after the hb */
671 put_hb_on_freelist(xi, (shared_buffer_t *)MmGetMdlVirtualAddress(curr_mdl) - 1);
672 }
673 else
674 {
675 //KdPrint((__DRIVER_NAME " returning page_buf %p with id %d\n", page_buf, page_buf->id));
676 if (curr_mdl != page_buf->mdl)
677 {
678 //KdPrint((__DRIVER_NAME " curr_mdl = %p, page_buf->mdl = %p\n", curr_mdl, page_buf->mdl));
679 IoFreeMdl(curr_mdl);
680 }
681 put_pb_on_freelist(xi, page_buf);
682 }
683 curr_mdl = next_mdl;
684 page_buf = next_buf;
685 }
687 NdisFreeNetBuffer(curr_nb);
688 InterlockedDecrement(&xi->rx_outstanding);
690 curr_nb = next_nb;
691 }
692 NdisFreeNetBufferList(curr_nbl);
693 curr_nbl = next_nbl;
694 }
696 if (!xi->rx_outstanding && xi->rx_shutting_down)
697 KeSetEvent(&xi->packet_returned_event, IO_NO_INCREMENT, FALSE);
699 //FUNCTION_EXIT();
700 }
702 /* We limit the number of packets per interrupt so that acks get a chance
703 under high rx load. The DPC is immediately re-scheduled */
704 //#define MAXIMUM_PACKETS_PER_INTERRUPT 32 /* this is calculated before large packet split */
705 //#define MAXIMUM_DATA_PER_INTERRUPT (MAXIMUM_PACKETS_PER_INTERRUPT * 1500) /* help account for large packets */
707 #define MAXIMUM_PACKETS_PER_INTERRUPT 2560 /* this is calculated before large packet split */
708 #define MAXIMUM_DATA_PER_INTERRUPT (MAXIMUM_PACKETS_PER_INTERRUPT * 1500) /* help account for large packets */
710 // Called at DISPATCH_LEVEL
711 BOOLEAN
712 XenNet_RxBufferCheck(struct xennet_info *xi)
713 {
714 RING_IDX cons, prod;
715 ULONG packet_count = 0;
716 ULONG packet_data = 0;
717 ULONG buffer_count = 0;
718 USHORT id;
719 int more_to_do = FALSE;
720 shared_buffer_t *page_buf;
721 //LIST_ENTRY rx_header_only_packet_list;
722 //PLIST_ENTRY entry;
723 //ULONG nbl_count = 0;
724 ULONG interim_packet_data = 0;
725 struct netif_extra_info *ei;
726 rx_context_t rc;
727 packet_info_t *pi = &xi->rxpi[KeGetCurrentProcessorNumber() & 0xff];
728 shared_buffer_t *head_buf = NULL;
729 shared_buffer_t *tail_buf = NULL;
730 shared_buffer_t *last_buf = NULL;
731 BOOLEAN extra_info_flag = FALSE;
732 BOOLEAN more_data_flag = FALSE;
733 BOOLEAN dont_set_event;
734 //FUNCTION_ENTER();
736 if (!xi->connected)
737 return FALSE; /* a delayed DPC could let this come through... just do nothing */
739 rc.first_nbl = NULL;
740 rc.last_nbl = NULL;
741 rc.packet_count = 0;
742 rc.nbl_count = 0;
744 /* get all the buffers off the ring as quickly as possible so the lock is held for a minimum amount of time */
745 KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
747 if (xi->rx_shutting_down)
748 {
749 /* there is a chance that our Dpc had been queued just before the shutdown... */
750 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
751 return FALSE;
752 }
754 if (xi->rx_partial_buf)
755 {
756 head_buf = xi->rx_partial_buf;
757 tail_buf = xi->rx_partial_buf;
758 while (tail_buf->next)
759 tail_buf = tail_buf->next;
760 more_data_flag = xi->rx_partial_more_data_flag;
761 extra_info_flag = xi->rx_partial_extra_info_flag;
762 xi->rx_partial_buf = NULL;
763 }
765 do {
766 prod = xi->rx.sring->rsp_prod;
767 KeMemoryBarrier(); /* Ensure we see responses up to 'prod'. */
769 for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAXIMUM_PACKETS_PER_INTERRUPT && packet_data < MAXIMUM_DATA_PER_INTERRUPT; cons++)
770 {
771 id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
772 page_buf = xi->rx_ring_pbs[id];
773 ASSERT(page_buf);
774 xi->rx_ring_pbs[id] = NULL;
775 xi->rx_id_free++;
776 memcpy(&page_buf->rsp, RING_GET_RESPONSE(&xi->rx, cons), max(sizeof(struct netif_rx_response), sizeof(struct netif_extra_info)));
777 if (!extra_info_flag)
778 {
779 if (page_buf->rsp.status <= 0
780 || page_buf->rsp.offset + page_buf->rsp.status > PAGE_SIZE)
781 {
782 KdPrint((__DRIVER_NAME " Error: rsp offset %d, size %d\n",
783 page_buf->rsp.offset, page_buf->rsp.status));
784 ASSERT(!extra_info_flag);
785 put_pb_on_freelist(xi, page_buf);
786 continue;
787 }
788 }
790 if (!head_buf)
791 {
792 head_buf = page_buf;
793 tail_buf = page_buf;
794 }
795 else
796 {
797 tail_buf->next = page_buf;
798 tail_buf = page_buf;
799 }
800 page_buf->next = NULL;
802 if (extra_info_flag)
803 {
804 ei = (struct netif_extra_info *)&page_buf->rsp;
805 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
806 }
807 else
808 {
809 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
810 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
811 interim_packet_data += page_buf->rsp.status;
812 }
814 if (!extra_info_flag && !more_data_flag)
815 {
816 last_buf = page_buf;
817 packet_count++;
818 packet_data += interim_packet_data;
819 interim_packet_data = 0;
820 }
821 buffer_count++;
822 }
823 xi->rx.rsp_cons = cons;
825 /* Give netback more buffers */
826 XenNet_FillRing(xi);
828 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
829 break;
831 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
832 if (!more_to_do)
833 {
834 xi->rx.sring->rsp_event = xi->rx.rsp_cons + 1;
835 KeMemoryBarrier();
836 more_to_do = RING_HAS_UNCONSUMED_RESPONSES(&xi->rx);
837 }
838 } while (more_to_do);
840 /* anything past last_buf belongs to an incomplete packet... */
841 if (last_buf && last_buf->next)
842 {
843 KdPrint((__DRIVER_NAME " Partial receive\n"));
844 xi->rx_partial_buf = last_buf->next;
845 xi->rx_partial_more_data_flag = more_data_flag;
846 xi->rx_partial_extra_info_flag = extra_info_flag;
847 last_buf->next = NULL;
848 }
850 KeReleaseSpinLockFromDpcLevel(&xi->rx_lock);
852 if (packet_count >= MAXIMUM_PACKETS_PER_INTERRUPT || packet_data >= MAXIMUM_DATA_PER_INTERRUPT)
853 {
854 /* fire again immediately */
855 KdPrint((__DRIVER_NAME " Dpc Duration Exceeded\n"));
856 /* we want the Dpc on the end of the queue. By definition we are already on the right CPU so we know the Dpc queue will be run immediately */
857 // KeSetImportanceDpc(&xi->rxtx_dpc, MediumImportance);
858 KeInsertQueueDpc(&xi->rxtx_dpc, NULL, NULL);
859 /* dont set an event in TX path */
860 dont_set_event = TRUE;
861 }
862 else
863 {
864 /* make sure the Dpc queue is run immediately next interrupt */
865 // KeSetImportanceDpc(&xi->rxtx_dpc, HighImportance);
866 /* set an event in TX path */
867 dont_set_event = FALSE;
868 }
870 /* make packets out of the buffers */
871 page_buf = head_buf;
872 extra_info_flag = FALSE;
873 more_data_flag = FALSE;
875 while (page_buf)
876 {
877 shared_buffer_t *next_buf = page_buf->next;
878 PMDL mdl;
880 page_buf->next = NULL;
881 if (extra_info_flag)
882 {
883 //KdPrint((__DRIVER_NAME " processing extra info\n"));
884 ei = (struct netif_extra_info *)&page_buf->rsp;
885 extra_info_flag = ei->flags & XEN_NETIF_EXTRA_FLAG_MORE;
886 switch (ei->type)
887 {
888 case XEN_NETIF_EXTRA_TYPE_GSO:
889 switch (ei->u.gso.type)
890 {
891 case XEN_NETIF_GSO_TYPE_TCPV4:
892 pi->mss = ei->u.gso.size;
893 //KdPrint((__DRIVER_NAME " mss = %d\n", pi->mss));
894 // TODO - put this assertion somewhere ASSERT(header_len + pi->mss <= PAGE_SIZE); // this limits MTU to PAGE_SIZE - XN_HEADER_LEN
895 break;
896 default:
897 KdPrint((__DRIVER_NAME " Unknown GSO type (%d) detected\n", ei->u.gso.type));
898 break;
899 }
900 break;
901 default:
902 KdPrint((__DRIVER_NAME " Unknown extra info type (%d) detected\n", ei->type));
903 break;
904 }
905 put_pb_on_freelist(xi, page_buf);
906 }
907 else
908 {
909 ASSERT(!page_buf->rsp.offset);
910 if (!more_data_flag) // handling the packet's 1st buffer
911 {
912 if (page_buf->rsp.flags & NETRXF_csum_blank)
913 pi->csum_blank = TRUE;
914 if (page_buf->rsp.flags & NETRXF_data_validated)
915 pi->data_validated = TRUE;
916 }
917 mdl = page_buf->mdl;
918 mdl->ByteCount = page_buf->rsp.status; //NdisAdjustBufferLength(mdl, page_buf->rsp.status);
919 //KdPrint((__DRIVER_NAME " buffer = %p, pb = %p\n", buffer, page_buf));
920 if (pi->first_pb)
921 {
922 ASSERT(pi->curr_pb);
923 //KdPrint((__DRIVER_NAME " additional buffer\n"));
924 pi->curr_pb->next = page_buf;
925 pi->curr_pb = page_buf;
926 ASSERT(pi->curr_mdl);
927 pi->curr_mdl->Next = mdl;
928 pi->curr_mdl = mdl;
929 }
930 else
931 {
932 pi->first_pb = page_buf;
933 pi->curr_pb = page_buf;
934 pi->first_mdl = mdl;
935 pi->curr_mdl = mdl;
936 }
937 //pi->mdl_count++;
938 extra_info_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_extra_info);
939 more_data_flag = (BOOLEAN)(page_buf->rsp.flags & NETRXF_more_data);
940 pi->total_length = pi->total_length + page_buf->rsp.status;
941 }
943 /* Packet done, add it to the list */
944 if (!more_data_flag && !extra_info_flag)
945 {
946 pi->curr_pb = pi->first_pb;
947 pi->curr_mdl = pi->first_mdl;
948 XenNet_MakePackets(xi, &rc, pi);
949 }
951 page_buf = next_buf;
952 }
953 ASSERT(!more_data_flag && !extra_info_flag);
955 if (rc.first_nbl)
956 {
957 NdisMIndicateReceiveNetBufferLists(xi->adapter_handle, rc.first_nbl,
958 NDIS_DEFAULT_PORT_NUMBER, rc.nbl_count,
959 NDIS_RECEIVE_FLAGS_DISPATCH_LEVEL
960 //| NDIS_RECEIVE_FLAGS_SINGLE_ETHER_TYPE
961 | NDIS_RECEIVE_FLAGS_PERFECT_FILTERED);
962 }
963 //FUNCTION_EXIT();
964 return dont_set_event;
965 }
967 /*
968 Free all Rx buffers (on halt, for example)
969 The ring must be stopped at this point.
970 */
972 static VOID
973 XenNet_PurgeRing(xennet_info_t *xi)
974 {
975 int i;
976 for (i = 0; i < NET_RX_RING_SIZE; i++)
977 {
978 if (xi->rx_ring_pbs[i] != NULL)
979 {
980 put_pb_on_freelist(xi, xi->rx_ring_pbs[i]);
981 xi->rx_ring_pbs[i] = NULL;
982 }
983 }
984 }
986 static VOID
987 XenNet_BufferFree(xennet_info_t *xi)
988 {
989 shared_buffer_t *sb;
991 XenNet_PurgeRing(xi);
993 /* because we are shutting down this won't allocate new ones */
994 while ((sb = get_pb_from_freelist(xi)) != NULL)
995 {
996 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
997 sb->gref, FALSE, (ULONG)'XNRX');
998 IoFreeMdl(sb->mdl);
999 NdisFreeMemory(sb->virtual, sizeof(shared_buffer_t), 0);
1000 NdisFreeMemory(sb, PAGE_SIZE, 0);
1002 while ((sb = get_hb_from_freelist(xi)) != NULL)
1004 IoFreeMdl(sb->mdl);
1005 NdisFreeMemory(sb, sizeof(shared_buffer_t) + MAX_ETH_HEADER_LENGTH + MAX_LOOKAHEAD_LENGTH, 0);
1009 VOID
1010 XenNet_BufferAlloc(xennet_info_t *xi)
1012 //NDIS_STATUS status;
1013 int i;
1015 xi->rx_id_free = NET_RX_RING_SIZE;
1016 xi->rx_outstanding = 0;
1018 for (i = 0; i < NET_RX_RING_SIZE; i++)
1020 xi->rx_ring_pbs[i] = NULL;
1024 VOID
1025 XenNet_RxResumeStart(xennet_info_t *xi)
1027 KIRQL old_irql;
1029 FUNCTION_ENTER();
1031 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1032 XenNet_PurgeRing(xi);
1033 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1035 FUNCTION_EXIT();
1038 VOID
1039 XenNet_RxResumeEnd(xennet_info_t *xi)
1041 KIRQL old_irql;
1043 FUNCTION_ENTER();
1045 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1046 //XenNet_BufferAlloc(xi);
1047 XenNet_FillRing(xi);
1048 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1050 FUNCTION_EXIT();
1053 BOOLEAN
1054 XenNet_RxInit(xennet_info_t *xi)
1056 NET_BUFFER_LIST_POOL_PARAMETERS nbl_pool_parameters;
1057 NET_BUFFER_POOL_PARAMETERS nb_pool_parameters;
1058 int ret;
1060 FUNCTION_ENTER();
1062 xi->rx_shutting_down = FALSE;
1063 KeInitializeSpinLock(&xi->rx_lock);
1064 KeInitializeEvent(&xi->packet_returned_event, SynchronizationEvent, FALSE);
1065 xi->rxpi = NdisAllocateMemoryWithTagPriority(xi->adapter_handle, sizeof(packet_info_t) * NdisSystemProcessorCount(), XENNET_POOL_TAG, NormalPoolPriority);
1066 if (!xi->rxpi)
1068 KdPrint(("NdisAllocateMemoryWithTagPriority failed\n"));
1069 return FALSE;
1071 NdisZeroMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount());
1073 ret = stack_new(&xi->rx_pb_stack, NET_RX_RING_SIZE * 4);
1074 if (!ret)
1076 FUNCTION_MSG("Failed to allocate rx_pb_stack\n");
1077 NdisFreeMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount(), 0);
1078 return FALSE;
1080 stack_new(&xi->rx_hb_stack, NET_RX_RING_SIZE * 4);
1081 if (!ret)
1083 FUNCTION_MSG("Failed to allocate rx_hb_stack\n");
1084 stack_delete(xi->rx_pb_stack, NULL, NULL);
1085 NdisFreeMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount(), 0);
1086 return FALSE;
1089 XenNet_BufferAlloc(xi);
1091 nbl_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
1092 nbl_pool_parameters.Header.Revision = NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
1093 nbl_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_LIST_POOL_PARAMETERS_REVISION_1;
1094 nbl_pool_parameters.ProtocolId = NDIS_PROTOCOL_ID_DEFAULT;
1095 nbl_pool_parameters.fAllocateNetBuffer = FALSE;
1096 nbl_pool_parameters.ContextSize = 0;
1097 nbl_pool_parameters.PoolTag = XENNET_POOL_TAG;
1098 nbl_pool_parameters.DataSize = 0; /* NET_BUFFERS are always allocated separately */
1100 xi->rx_nbl_pool = NdisAllocateNetBufferListPool(xi->adapter_handle, &nbl_pool_parameters);
1101 if (!xi->rx_nbl_pool)
1103 KdPrint(("NdisAllocateNetBufferListPool failed\n"));
1104 return FALSE;
1107 nb_pool_parameters.Header.Type = NDIS_OBJECT_TYPE_DEFAULT;
1108 nb_pool_parameters.Header.Revision = NET_BUFFER_POOL_PARAMETERS_REVISION_1;
1109 nb_pool_parameters.Header.Size = NDIS_SIZEOF_NET_BUFFER_POOL_PARAMETERS_REVISION_1;
1110 nb_pool_parameters.PoolTag = XENNET_POOL_TAG;
1111 nb_pool_parameters.DataSize = 0; /* the buffers come from the ring */
1112 xi->rx_nb_pool = NdisAllocateNetBufferPool(xi->adapter_handle, &nb_pool_parameters);
1113 if (!xi->rx_nb_pool)
1115 KdPrint(("NdisAllocateNetBufferPool (rx_nb_pool) failed\n"));
1116 return FALSE;
1119 XenNet_FillRing(xi);
1121 FUNCTION_EXIT();
1123 return TRUE;
1126 BOOLEAN
1127 XenNet_RxShutdown(xennet_info_t *xi)
1129 KIRQL old_irql;
1130 //PNDIS_PACKET packet;
1131 UNREFERENCED_PARAMETER(xi);
1133 FUNCTION_ENTER();
1135 KeAcquireSpinLock(&xi->rx_lock, &old_irql);
1136 xi->rx_shutting_down = TRUE;
1137 KeReleaseSpinLock(&xi->rx_lock, old_irql);
1139 KeFlushQueuedDpcs();
1141 while (xi->rx_outstanding)
1143 KdPrint((__DRIVER_NAME " Waiting for all packets to be returned\n"));
1144 KeWaitForSingleObject(&xi->packet_returned_event, Executive, KernelMode, FALSE, NULL);
1147 NdisFreeMemory(xi->rxpi, sizeof(packet_info_t) * NdisSystemProcessorCount(), 0);
1149 XenNet_BufferFree(xi);
1152 stack_delete(xi->rx_pb_stack, NULL, NULL);
1153 stack_delete(xi->rx_hb_stack, NULL, NULL);
1155 NdisFreeNetBufferPool(xi->rx_nb_pool);
1156 NdisFreeNetBufferListPool(xi->rx_nbl_pool);
1158 FUNCTION_EXIT();
1160 return TRUE;