win-pvdrivers

annotate xennet/xennet_tx.c @ 821:9c0c4210b778

Fix xennet build under Windows 2000
Fix xenvbd install under Windows 2000
author James Harper <james.harper@bendigoit.com.au>
date Sat Oct 16 20:03:30 2010 +1100 (2010-10-16)
parents 467005e7f509
children 254b8424e23b
rev   line source
james@204 1 /*
james@204 2 PV Net Driver for Windows Xen HVM Domains
james@204 3 Copyright (C) 2007 James Harper
james@204 4 Copyright (C) 2007 Andrew Grover <andy.grover@oracle.com>
james@204 5
james@204 6 This program is free software; you can redistribute it and/or
james@204 7 modify it under the terms of the GNU General Public License
james@204 8 as published by the Free Software Foundation; either version 2
james@204 9 of the License, or (at your option) any later version.
james@204 10
james@204 11 This program is distributed in the hope that it will be useful,
james@204 12 but WITHOUT ANY WARRANTY; without even the implied warranty of
james@204 13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
james@204 14 GNU General Public License for more details.
james@204 15
james@204 16 You should have received a copy of the GNU General Public License
james@204 17 along with this program; if not, write to the Free Software
james@204 18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
james@204 19 */
james@204 20
james@204 21 #include "xennet.h"
james@204 22
james@716 23 /* Not really necessary but keeps PREfast happy */
james@821 24 #if (NTDDI_VERSION >= NTDDI_WINXP)
james@716 25 static KDEFERRED_ROUTINE XenNet_TxBufferGC;
james@821 26 #endif
james@716 27
james@204 28 static USHORT
james@206 29 get_id_from_freelist(struct xennet_info *xi)
james@204 30 {
james@535 31 ASSERT(xi->tx_id_free);
james@204 32 xi->tx_id_free--;
james@254 33
james@204 34 return xi->tx_id_list[xi->tx_id_free];
james@204 35 }
james@204 36
james@204 37 static VOID
james@206 38 put_id_on_freelist(struct xennet_info *xi, USHORT id)
james@204 39 {
james@204 40 xi->tx_id_list[xi->tx_id_free] = id;
james@204 41 xi->tx_id_free++;
james@204 42 }
james@204 43
james@535 44 #define SWAP_USHORT(x) (USHORT)((((x & 0xFF) << 8)|((x >> 8) & 0xFF)))
james@535 45
james@780 46 static __forceinline struct netif_tx_request *
james@780 47 XenNet_PutCbOnRing(struct xennet_info *xi, PVOID coalesce_buf, ULONG length, grant_ref_t gref)
james@780 48 {
james@780 49 struct netif_tx_request *tx;
james@780 50 tx = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
james@780 51 xi->tx.req_prod_pvt++;
james@780 52 xi->tx_ring_free--;
james@780 53 tx->id = get_id_from_freelist(xi);
james@780 54 ASSERT(xi->tx_shadows[tx->id].gref == INVALID_GRANT_REF);
james@780 55 ASSERT(!xi->tx_shadows[tx->id].cb);
james@780 56 xi->tx_shadows[tx->id].cb = coalesce_buf;
james@790 57 tx->gref = xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0, (ULONG)(MmGetPhysicalAddress(coalesce_buf).QuadPart >> PAGE_SHIFT), FALSE, gref, (ULONG)'XNTX');
james@780 58 xi->tx_shadows[tx->id].gref = tx->gref;
james@780 59 tx->offset = 0;
james@780 60 tx->size = (USHORT)length;
james@780 61 ASSERT(tx->offset + tx->size <= PAGE_SIZE);
james@780 62 ASSERT(tx->size);
james@780 63 return tx;
james@780 64 }
james@780 65
james@205 66 /* Called at DISPATCH_LEVEL with tx_lock held */
andy@229 67 /*
andy@229 68 * Send one NDIS_PACKET. This may involve multiple entries on TX ring.
andy@229 69 */
andy@216 70 static BOOLEAN
andy@216 71 XenNet_HWSendPacket(struct xennet_info *xi, PNDIS_PACKET packet)
andy@216 72 {
james@535 73 struct netif_tx_request *tx0 = NULL;
james@535 74 struct netif_tx_request *txN = NULL;
james@254 75 struct netif_extra_info *ei = NULL;
andy@216 76 PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
james@436 77 ULONG mss = 0;
andy@232 78 uint16_t flags = NETTXF_more_data;
james@247 79 packet_info_t pi;
james@254 80 BOOLEAN ndis_lso = FALSE;
james@254 81 BOOLEAN xen_gso = FALSE;
james@780 82 ULONG remaining;
james@535 83 ULONG parse_result;
james@780 84 ULONG frags = 0;
james@780 85 BOOLEAN coalesce_required = FALSE;
james@780 86 PVOID coalesce_buf;
james@780 87 ULONG coalesce_remaining = 0;
james@780 88 grant_ref_t gref;
james@780 89 ULONG tx_length = 0;
james@535 90
james@535 91 //FUNCTION_ENTER();
james@780 92
james@790 93 gref = xi->vectors.GntTbl_GetRef(xi->vectors.context, (ULONG)'XNTX');
james@780 94 if (gref == INVALID_GRANT_REF)
james@780 95 {
james@780 96 KdPrint((__DRIVER_NAME " out of grefs\n"));
james@780 97 return FALSE;
james@780 98 }
james@780 99 coalesce_buf = NdisAllocateFromNPagedLookasideList(&xi->tx_lookaside_list);
james@780 100 if (!coalesce_buf)
james@780 101 {
james@790 102 xi->vectors.GntTbl_PutRef(xi->vectors.context, gref, (ULONG)'XNTX');
james@780 103 KdPrint((__DRIVER_NAME " out of memory\n"));
james@780 104 return FALSE;
james@780 105 }
james@443 106 XenNet_ClearPacketInfo(&pi);
james@536 107 NdisQueryPacket(packet, NULL, (PUINT)&pi.mdl_count, &pi.first_buffer, (PUINT)&pi.total_length);
james@780 108
james@780 109 pi.curr_mdl_offset = 0;
james@780 110 pi.curr_buffer = pi.first_buffer;
james@780 111 remaining = min(pi.total_length, PAGE_SIZE);
james@780 112 while (remaining) /* this much gets put in the header */
james@596 113 {
james@780 114 ULONG length = XenNet_QueryData(&pi, remaining);
james@780 115 remaining -= length;
james@780 116 XenNet_EatData(&pi, length);
james@780 117 }
james@780 118 frags++;
james@780 119 if (pi.total_length > PAGE_SIZE) /* these are the frags we care about */
james@780 120 {
james@780 121 remaining = pi.total_length - PAGE_SIZE;
james@780 122 while (remaining)
james@596 123 {
james@780 124 ULONG length = XenNet_QueryData(&pi, PAGE_SIZE);
james@780 125 if (length != 0)
james@780 126 {
james@780 127 frags++;
james@780 128 if (frags > LINUX_MAX_SG_ELEMENTS)
james@780 129 break; /* worst case there could be hundreds of fragments - leave the loop now */
james@780 130 }
james@780 131 remaining -= length;
james@780 132 XenNet_EatData(&pi, length);
james@596 133 }
james@780 134 }
james@780 135 if (frags > LINUX_MAX_SG_ELEMENTS)
james@780 136 {
james@780 137 frags = LINUX_MAX_SG_ELEMENTS;
james@780 138 coalesce_required = TRUE;
james@780 139 }
james@780 140
james@780 141 /* if we have enough space on the ring then we have enough id's so no need to check for that */
james@780 142 if (xi->tx_ring_free < frags + 1)
james@780 143 {
james@790 144 xi->vectors.GntTbl_PutRef(xi->vectors.context, gref, (ULONG)'XNTX');
james@780 145 NdisFreeToNPagedLookasideList(&xi->tx_lookaside_list, coalesce_buf);
james@780 146 //KdPrint((__DRIVER_NAME " Full on send - ring full\n"));
james@780 147 return FALSE;
james@596 148 }
james@596 149
james@780 150 parse_result = XenNet_ParsePacketHeader(&pi, coalesce_buf, PAGE_SIZE);
james@780 151 remaining = pi.total_length - pi.header_length;
james@780 152
james@436 153 if (NDIS_GET_PACKET_PROTOCOL_TYPE(packet) == NDIS_PROTOCOL_ID_TCP_IP)
james@436 154 {
james@436 155 csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
james@436 156 packet, TcpIpChecksumPacketInfo);
james@436 157 if (csum_info->Transmit.NdisPacketChecksumV4)
james@436 158 {
james@436 159 if (csum_info->Transmit.NdisPacketIpChecksum && !xi->setting_csum.V4Transmit.IpChecksum)
james@436 160 {
james@436 161 KdPrint((__DRIVER_NAME " IpChecksum not enabled\n"));
james@439 162 //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE);
james@439 163 //return TRUE;
james@436 164 }
james@439 165 if (csum_info->Transmit.NdisPacketTcpChecksum)
james@436 166 {
james@439 167 if (xi->setting_csum.V4Transmit.TcpChecksum)
james@439 168 {
james@439 169 flags |= NETTXF_csum_blank | NETTXF_data_validated;
james@439 170 }
james@439 171 else
james@439 172 {
james@439 173 KdPrint((__DRIVER_NAME " TcpChecksum not enabled\n"));
james@439 174 //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE);
james@439 175 //return TRUE;
james@439 176 }
james@436 177 }
james@493 178 else if (csum_info->Transmit.NdisPacketUdpChecksum)
james@436 179 {
james@439 180 if (xi->setting_csum.V4Transmit.UdpChecksum)
james@439 181 {
james@439 182 flags |= NETTXF_csum_blank | NETTXF_data_validated;
james@439 183 }
james@439 184 else
james@439 185 {
james@439 186 KdPrint((__DRIVER_NAME " UdpChecksum not enabled\n"));
james@439 187 //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE);
james@439 188 //return TRUE;
james@439 189 }
james@436 190 }
james@436 191 }
james@493 192 else if (csum_info->Transmit.NdisPacketChecksumV6)
james@493 193 {
james@493 194 KdPrint((__DRIVER_NAME " NdisPacketChecksumV6 not supported\n"));
james@493 195 //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE);
james@493 196 //return TRUE;
james@493 197 }
james@436 198 }
james@780 199
james@248 200 mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo));
james@247 201
james@674 202 if (mss && parse_result == PARSE_OK)
james@436 203 {
james@436 204 if (NDIS_GET_PACKET_PROTOCOL_TYPE(packet) != NDIS_PROTOCOL_ID_TCP_IP)
james@436 205 {
james@436 206 KdPrint((__DRIVER_NAME " mss specified when packet is not NDIS_PROTOCOL_ID_TCP_IP\n"));
james@436 207 }
james@254 208 ndis_lso = TRUE;
james@436 209 if (mss > xi->setting_max_offload)
james@436 210 {
james@436 211 KdPrint((__DRIVER_NAME " Requested MSS (%d) larger than allowed MSS (%d)\n", mss, xi->setting_max_offload));
james@535 212 //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_FAILURE);
james@535 213 //FUNCTION_EXIT();
james@436 214 return TRUE;
james@436 215 }
james@436 216 }
james@247 217
james@254 218 if (ndis_lso)
james@535 219 {
james@674 220 flags |= NETTXF_csum_blank | NETTXF_data_validated; /* these may be implied but not specified when lso is used*/
james@674 221 if (pi.tcp_length >= mss)
james@254 222 {
james@674 223 flags |= NETTXF_extra_info;
james@674 224 xen_gso = TRUE;
james@436 225 }
james@436 226 else
james@436 227 {
james@674 228 KdPrint((__DRIVER_NAME " large send specified when tcp_length < mss\n"));
james@254 229 }
andy@229 230 }
andy@229 231
james@535 232 /*
james@535 233 * See io/netif.h. Must put (A) 1st request, then (B) optional extra_info, then
james@535 234 * (C) rest of requests on the ring. Only (A) has csum flags.
james@535 235 */
james@254 236
andy@229 237 /* (A) */
james@780 238 tx0 = XenNet_PutCbOnRing(xi, coalesce_buf, pi.header_length, gref);
james@780 239 ASSERT(tx0); /* this will never happen */
james@780 240 tx0->flags = flags;
james@780 241 tx_length += pi.header_length;
james@780 242
james@780 243 /* even though we haven't reported that we are capable of it, LSO demands that we calculate the IP Header checksum */
james@780 244 if (ndis_lso)
james@535 245 {
james@780 246 XenNet_SumIpHeader(coalesce_buf, pi.ip4_header_length);
james@535 247 }
james@535 248 txN = tx0;
andy@229 249
andy@229 250 /* (B) */
james@254 251 if (xen_gso)
andy@229 252 {
james@254 253 ASSERT(flags & NETTXF_extra_info);
andy@229 254 ei = (struct netif_extra_info *)RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
james@766 255 //KdPrint((__DRIVER_NAME " pos = %d\n", xi->tx.req_prod_pvt));
james@766 256 xi->tx.req_prod_pvt++;
james@559 257 xi->tx_ring_free--;
andy@229 258 ei->type = XEN_NETIF_EXTRA_TYPE_GSO;
james@237 259 ei->flags = 0;
james@254 260 ei->u.gso.size = (USHORT)mss;
andy@229 261 ei->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
andy@229 262 ei->u.gso.pad = 0;
andy@229 263 ei->u.gso.features = 0;
andy@229 264 }
andy@229 265
james@780 266 ASSERT(xi->config_sg || !remaining);
james@780 267
james@780 268 /* (C) - only if data is remaining */
james@780 269 coalesce_buf = NULL;
james@780 270 while (remaining > 0)
andy@216 271 {
james@780 272 ULONG length;
james@780 273 PFN_NUMBER pfn;
james@780 274
james@780 275 ASSERT(pi.curr_buffer);
james@780 276 if (coalesce_required)
james@596 277 {
james@780 278 PVOID va;
james@780 279 if (!coalesce_buf)
james@766 280 {
james@790 281 gref = xi->vectors.GntTbl_GetRef(xi->vectors.context, (ULONG)'XNTX');
james@780 282 if (gref == INVALID_GRANT_REF)
james@780 283 {
james@780 284 KdPrint((__DRIVER_NAME " out of grefs - partial send\n"));
james@780 285 break;
james@780 286 }
james@780 287 coalesce_buf = NdisAllocateFromNPagedLookasideList(&xi->tx_lookaside_list);
james@780 288 if (!coalesce_buf)
james@780 289 {
james@790 290 xi->vectors.GntTbl_PutRef(xi->vectors.context, gref, (ULONG)'XNTX');
james@780 291 KdPrint((__DRIVER_NAME " out of memory - partial send\n"));
james@780 292 break;
james@780 293 }
james@780 294 coalesce_remaining = min(PAGE_SIZE, remaining);
james@766 295 }
james@780 296 length = XenNet_QueryData(&pi, coalesce_remaining);
james@780 297 va = NdisBufferVirtualAddressSafe(pi.curr_buffer, LowPagePriority);
james@780 298 if (!va)
james@766 299 {
james@780 300 KdPrint((__DRIVER_NAME " failed to map buffer va - partial send\n"));
james@780 301 coalesce_remaining = 0;
james@780 302 remaining -= min(PAGE_SIZE, remaining);
james@780 303 NdisFreeToNPagedLookasideList(&xi->tx_lookaside_list, coalesce_buf);
james@766 304 }
james@766 305 else
james@766 306 {
james@780 307 memcpy((PUCHAR)coalesce_buf + min(PAGE_SIZE, remaining) - coalesce_remaining, (PUCHAR)va + pi.curr_mdl_offset, length);
james@780 308 coalesce_remaining -= length;
james@766 309 }
james@596 310 }
james@780 311 else
james@780 312 {
james@780 313 length = XenNet_QueryData(&pi, PAGE_SIZE);
james@780 314 }
james@780 315 if (!length || coalesce_remaining) /* sometimes there are zero length buffers... */
james@780 316 {
james@780 317 XenNet_EatData(&pi, length); /* do this so we actually move to the next buffer */
james@780 318 continue;
james@780 319 }
james@780 320
james@780 321 if (coalesce_buf)
james@780 322 {
james@780 323 if (remaining)
james@780 324 {
james@780 325 txN = XenNet_PutCbOnRing(xi, coalesce_buf, min(PAGE_SIZE, remaining), gref);
james@780 326 ASSERT(txN);
james@780 327 coalesce_buf = NULL;
james@780 328 remaining -= min(PAGE_SIZE, remaining);
james@780 329 tx_length += min(PAGE_SIZE, remaining);
james@780 330 }
james@780 331 }
james@780 332 else
james@780 333 {
james@780 334 ULONG offset;
james@780 335
james@790 336 gref = xi->vectors.GntTbl_GetRef(xi->vectors.context, (ULONG)'XNTX');
james@780 337 if (gref == INVALID_GRANT_REF)
james@780 338 {
james@780 339 KdPrint((__DRIVER_NAME " out of grefs - partial send\n"));
james@780 340 break;
james@780 341 }
james@780 342 txN = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
james@780 343 xi->tx.req_prod_pvt++;
james@780 344 xi->tx_ring_free--;
james@780 345 txN->id = get_id_from_freelist(xi);
james@780 346 ASSERT(!xi->tx_shadows[txN->id].cb);
james@780 347 offset = MmGetMdlByteOffset(pi.curr_buffer) + pi.curr_mdl_offset;
james@780 348 pfn = MmGetMdlPfnArray(pi.curr_buffer)[offset >> PAGE_SHIFT];
james@780 349 txN->offset = (USHORT)offset & (PAGE_SIZE - 1);
james@790 350 txN->gref = xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0, (ULONG)pfn, FALSE, gref, (ULONG)'XNTX');
james@780 351 ASSERT(xi->tx_shadows[txN->id].gref == INVALID_GRANT_REF);
james@780 352 xi->tx_shadows[txN->id].gref = txN->gref;
james@780 353 //ASSERT(sg->Elements[sg_element].Length > sg_offset);
james@780 354 txN->size = (USHORT)length;
james@780 355 ASSERT(txN->offset + txN->size <= PAGE_SIZE);
james@780 356 ASSERT(txN->size);
james@780 357 ASSERT(txN->gref != INVALID_GRANT_REF);
james@780 358 remaining -= length;
james@780 359 tx_length += length;
james@780 360 }
james@780 361 tx0->size = tx0->size + txN->size;
james@780 362 txN->flags = NETTXF_more_data;
james@780 363 XenNet_EatData(&pi, length);
andy@216 364 }
james@535 365 txN->flags &= ~NETTXF_more_data;
james@562 366 ASSERT(tx0->size == pi.total_length);
james@559 367 ASSERT(!xi->tx_shadows[txN->id].packet);
james@535 368 xi->tx_shadows[txN->id].packet = packet;
andy@229 369
james@254 370 if (ndis_lso)
james@252 371 {
james@535 372 //KdPrint((__DRIVER_NAME " TcpLargeSendPacketInfo = %d\n", pi.tcp_length));
james@780 373 NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo) = UlongToPtr(tx_length - MAX_ETH_HEADER_LENGTH - pi.ip4_header_length - pi.tcp_header_length);
james@252 374 }
james@252 375
james@411 376 xi->stat_tx_ok++;
james@411 377
james@535 378 //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
james@535 379 //FUNCTION_EXIT();
james@537 380 xi->tx_outstanding++;
andy@216 381 return TRUE;
andy@216 382 }
andy@216 383
andy@216 384 /* Called at DISPATCH_LEVEL with tx_lock held */
andy@216 385
james@204 386 static VOID
james@204 387 XenNet_SendQueuedPackets(struct xennet_info *xi)
james@204 388 {
james@204 389 PLIST_ENTRY entry;
james@204 390 PNDIS_PACKET packet;
james@204 391 int notify;
james@204 392
james@459 393 //FUNCTION_ENTER();
james@459 394
james@536 395 if (xi->device_state->suspend_resume_state_pdo != SR_STATE_RUNNING)
james@536 396 return;
james@536 397
james@204 398 entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
james@204 399 /* if empty, the above returns head*, not NULL */
james@204 400 while (entry != &xi->tx_waiting_pkt_list)
james@204 401 {
james@204 402 packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
james@535 403 if (!XenNet_HWSendPacket(xi, packet))
james@252 404 {
james@780 405 //KdPrint((__DRIVER_NAME " No room for packet\n"));
james@252 406 InsertHeadList(&xi->tx_waiting_pkt_list, entry);
james@206 407 break;
james@252 408 }
james@204 409 entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
james@204 410 }
james@204 411
james@204 412 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xi->tx, notify);
james@204 413 if (notify)
james@204 414 {
james@271 415 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->event_channel);
james@204 416 }
james@459 417 //FUNCTION_EXIT();
james@204 418 }
james@204 419
james@535 420 //ULONG packets_outstanding = 0;
james@204 421 // Called at DISPATCH_LEVEL
james@716 422 static VOID
james@454 423 XenNet_TxBufferGC(PKDPC dpc, PVOID context, PVOID arg1, PVOID arg2)
james@204 424 {
james@454 425 struct xennet_info *xi = context;
james@204 426 RING_IDX cons, prod;
james@535 427 PNDIS_PACKET head = NULL, tail = NULL;
james@535 428 PNDIS_PACKET packet;
james@775 429 ULONG tx_packets = 0;
james@204 430
james@454 431 UNREFERENCED_PARAMETER(dpc);
james@454 432 UNREFERENCED_PARAMETER(arg1);
james@454 433 UNREFERENCED_PARAMETER(arg2);
james@454 434
james@459 435 //FUNCTION_ENTER();
james@459 436
james@790 437 if (!xi->connected)
james@790 438 return; /* a delayed DPC could let this come through... just do nothing */
james@204 439 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
james@204 440
james@204 441 KeAcquireSpinLockAtDpcLevel(&xi->tx_lock);
james@204 442
james@603 443 if (xi->tx_shutting_down && !xi->tx_outstanding)
james@586 444 {
james@586 445 /* there is a chance that our Dpc had been queued just before the shutdown... */
james@635 446 KeSetEvent(&xi->tx_idle_event, IO_NO_INCREMENT, FALSE);
james@586 447 KeReleaseSpinLockFromDpcLevel(&xi->tx_lock);
james@586 448 return;
james@586 449 }
james@586 450
james@204 451 do {
james@204 452 prod = xi->tx.sring->rsp_prod;
james@254 453 KeMemoryBarrier(); /* Ensure we see responses up to 'rsp_prod'. */
james@204 454
james@205 455 for (cons = xi->tx.rsp_cons; cons != prod; cons++)
james@205 456 {
james@204 457 struct netif_tx_response *txrsp;
james@766 458 tx_shadow_t *shadow;
james@766 459
james@204 460 txrsp = RING_GET_RESPONSE(&xi->tx, cons);
james@559 461
james@559 462 xi->tx_ring_free++;
james@559 463
james@766 464 if (txrsp->status == NETIF_RSP_NULL)
james@766 465 {
james@535 466 continue;
james@766 467 }
james@209 468
james@766 469 shadow = &xi->tx_shadows[txrsp->id];
james@766 470 if (shadow->cb)
james@247 471 {
james@780 472 NdisFreeToNPagedLookasideList(&xi->tx_lookaside_list, shadow->cb);
james@766 473 shadow->cb = NULL;
james@247 474 }
james@535 475
james@766 476 if (shadow->gref != INVALID_GRANT_REF)
james@535 477 {
james@766 478 xi->vectors.GntTbl_EndAccess(xi->vectors.context,
james@790 479 shadow->gref, FALSE, (ULONG)'XNTX');
james@766 480 shadow->gref = INVALID_GRANT_REF;
james@766 481 }
james@766 482
james@766 483 if (shadow->packet)
james@766 484 {
james@766 485 packet = shadow->packet;
james@535 486 *(PNDIS_PACKET *)&packet->MiniportReservedEx[0] = NULL;
james@535 487 if (head)
james@535 488 *(PNDIS_PACKET *)&tail->MiniportReservedEx[0] = packet;
james@535 489 else
james@535 490 head = packet;
james@535 491 tail = packet;
james@766 492 shadow->packet = NULL;
james@535 493 }
james@535 494 put_id_on_freelist(xi, txrsp->id);
james@204 495 }
james@204 496
james@204 497 xi->tx.rsp_cons = prod;
james@766 498 /* resist the temptation to set the event more than +1... it breaks things */
james@766 499 xi->tx.sring->rsp_event = prod + 1;
james@411 500 KeMemoryBarrier();
james@411 501 } while (prod != xi->tx.sring->rsp_prod);
james@204 502
james@204 503 /* if queued packets, send them now */
james@603 504 if (!xi->tx_shutting_down)
james@576 505 XenNet_SendQueuedPackets(xi);
james@204 506
james@205 507 KeReleaseSpinLockFromDpcLevel(&xi->tx_lock);
james@205 508
james@775 509 /* must be done without holding any locks */
james@535 510 while (head)
james@535 511 {
james@535 512 packet = (PNDIS_PACKET)head;
james@535 513 head = *(PNDIS_PACKET *)&packet->MiniportReservedEx[0];
james@535 514 NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_SUCCESS);
james@775 515 tx_packets++;
james@535 516 }
james@536 517
james@775 518 /* must be done after we have truly given back all packets */
james@775 519 KeAcquireSpinLockAtDpcLevel(&xi->tx_lock);
james@775 520 xi->tx_outstanding -= tx_packets;
james@775 521 if (!xi->tx_outstanding && xi->tx_shutting_down)
james@775 522 KeSetEvent(&xi->tx_idle_event, IO_NO_INCREMENT, FALSE);
james@775 523 KeReleaseSpinLockFromDpcLevel(&xi->tx_lock);
james@775 524
james@536 525 if (xi->device_state->suspend_resume_state_pdo == SR_STATE_SUSPENDING
james@536 526 && xi->device_state->suspend_resume_state_fdo != SR_STATE_SUSPENDING
james@536 527 && xi->tx_id_free == NET_TX_RING_SIZE)
james@536 528 {
james@536 529 KdPrint((__DRIVER_NAME " Setting SR_STATE_SUSPENDING\n"));
james@536 530 xi->device_state->suspend_resume_state_fdo = SR_STATE_SUSPENDING;
james@536 531 KdPrint((__DRIVER_NAME " Notifying event channel %d\n", xi->device_state->pdo_event_channel));
james@536 532 xi->vectors.EvtChn_Notify(xi->vectors.context, xi->device_state->pdo_event_channel);
james@536 533 }
james@536 534
james@459 535 //FUNCTION_EXIT();
james@204 536 }
james@204 537
james@343 538 // called at <= DISPATCH_LEVEL
andy@369 539 VOID DDKAPI
james@204 540 XenNet_SendPackets(
james@204 541 IN NDIS_HANDLE MiniportAdapterContext,
james@204 542 IN PPNDIS_PACKET PacketArray,
james@204 543 IN UINT NumberOfPackets
james@204 544 )
james@204 545 {
james@204 546 struct xennet_info *xi = MiniportAdapterContext;
james@206 547 PNDIS_PACKET packet;
james@204 548 UINT i;
james@204 549 PLIST_ENTRY entry;
james@204 550 KIRQL OldIrql;
james@204 551
james@459 552 //FUNCTION_ENTER();
james@459 553
james@516 554 if (xi->inactive)
james@516 555 {
james@516 556 for (i = 0; i < NumberOfPackets; i++)
james@516 557 {
james@535 558 NdisMSendComplete(xi->adapter_handle, PacketArray[i], NDIS_STATUS_FAILURE);
james@516 559 }
james@516 560 return;
james@516 561 }
james@516 562
james@204 563 KeAcquireSpinLock(&xi->tx_lock, &OldIrql);
james@204 564
james@204 565 for (i = 0; i < NumberOfPackets; i++)
james@204 566 {
james@206 567 packet = PacketArray[i];
james@206 568 ASSERT(packet);
james@206 569 *(ULONG *)&packet->MiniportReservedEx = 0;
james@206 570 entry = (PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)];
james@204 571 InsertTailList(&xi->tx_waiting_pkt_list, entry);
james@204 572 }
james@205 573
james@536 574 XenNet_SendQueuedPackets(xi);
james@205 575
james@204 576 KeReleaseSpinLock(&xi->tx_lock, OldIrql);
james@411 577
james@459 578 //FUNCTION_EXIT();
james@343 579 }
james@204 580
james@343 581 VOID
james@596 582 XenNet_CancelSendPackets(
james@596 583 NDIS_HANDLE MiniportAdapterContext,
james@596 584 PVOID CancelId)
james@596 585 {
james@596 586 struct xennet_info *xi = MiniportAdapterContext;
james@596 587 KIRQL old_irql;
james@596 588 PLIST_ENTRY entry;
james@596 589 PNDIS_PACKET packet;
james@596 590 PNDIS_PACKET head = NULL, tail = NULL;
james@775 591 BOOLEAN result;
james@596 592
james@596 593 FUNCTION_ENTER();
james@596 594
james@596 595 KeAcquireSpinLock(&xi->tx_lock, &old_irql);
james@596 596
james@596 597 entry = xi->tx_waiting_pkt_list.Flink;
james@596 598 while (entry != &xi->tx_waiting_pkt_list)
james@596 599 {
james@596 600 packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
james@596 601 entry = entry->Flink;
james@596 602 if (NDIS_GET_PACKET_CANCEL_ID(packet) == CancelId)
james@596 603 {
james@775 604 KdPrint((__DRIVER_NAME " Found packet to cancel %p\n", packet));
james@775 605 result = RemoveEntryList((PLIST_ENTRY)&packet->MiniportReservedEx[sizeof(PVOID)]);
james@775 606 ASSERT(result);
james@596 607 *(PNDIS_PACKET *)&packet->MiniportReservedEx[0] = NULL;
james@596 608 if (head)
james@596 609 *(PNDIS_PACKET *)&tail->MiniportReservedEx[0] = packet;
james@596 610 else
james@596 611 head = packet;
james@596 612 tail = packet;
james@596 613 }
james@596 614 }
james@596 615
james@596 616 KeReleaseSpinLock(&xi->tx_lock, old_irql);
james@596 617
james@596 618 while (head)
james@596 619 {
james@596 620 packet = (PNDIS_PACKET)head;
james@596 621 head = *(PNDIS_PACKET *)&packet->MiniportReservedEx[0];
james@775 622 KdPrint((__DRIVER_NAME " NdisMSendComplete(%p)\n", packet));
james@596 623 NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_REQUEST_ABORTED);
james@596 624 }
james@596 625
james@596 626 FUNCTION_EXIT();
james@596 627 }
james@596 628
james@596 629 VOID
james@343 630 XenNet_TxResumeStart(xennet_info_t *xi)
james@343 631 {
james@535 632 UNREFERENCED_PARAMETER(xi);
james@343 633
james@536 634 FUNCTION_ENTER();
james@536 635 /* nothing to do here - all packets were already sent */
james@536 636 FUNCTION_EXIT();
james@343 637 }
james@343 638
james@343 639 VOID
james@343 640 XenNet_TxResumeEnd(xennet_info_t *xi)
james@343 641 {
james@343 642 KIRQL old_irql;
james@343 643
james@536 644 FUNCTION_ENTER();
james@536 645
james@343 646 KeAcquireSpinLock(&xi->tx_lock, &old_irql);
james@343 647 XenNet_SendQueuedPackets(xi);
james@343 648 KeReleaseSpinLock(&xi->tx_lock, old_irql);
james@536 649
james@536 650 FUNCTION_EXIT();
james@204 651 }
james@204 652
james@204 653 BOOLEAN
james@204 654 XenNet_TxInit(xennet_info_t *xi)
james@204 655 {
james@780 656 USHORT i;
james@204 657
james@253 658 KeInitializeSpinLock(&xi->tx_lock);
james@454 659 KeInitializeDpc(&xi->tx_dpc, XenNet_TxBufferGC, xi);
james@454 660 /* dpcs are only serialised to a single processor */
james@454 661 KeSetTargetProcessorDpc(&xi->tx_dpc, 0);
james@457 662 //KeSetImportanceDpc(&xi->tx_dpc, HighImportance);
james@535 663 InitializeListHead(&xi->tx_waiting_pkt_list);
james@535 664
james@537 665 KeInitializeEvent(&xi->tx_idle_event, SynchronizationEvent, FALSE);
james@619 666 xi->tx_shutting_down = FALSE;
james@537 667 xi->tx_outstanding = 0;
james@535 668 xi->tx_ring_free = NET_TX_RING_SIZE;
james@596 669
james@780 670 NdisInitializeNPagedLookasideList(&xi->tx_lookaside_list, NULL, NULL, 0,
james@780 671 PAGE_SIZE, XENNET_POOL_TAG, 0);
james@253 672
james@204 673 xi->tx_id_free = 0;
james@204 674 for (i = 0; i < NET_TX_RING_SIZE; i++)
james@204 675 {
james@766 676 xi->tx_shadows[i].gref = INVALID_GRANT_REF;
james@766 677 xi->tx_shadows[i].cb = NULL;
james@206 678 put_id_on_freelist(xi, i);
james@206 679 }
james@250 680
james@204 681 return TRUE;
james@204 682 }
james@204 683
james@254 684 /*
james@254 685 The ring is completely closed down now. We just need to empty anything left
james@535 686 on our freelists and harvest anything left on the rings.
james@254 687 */
james@254 688
james@204 689 BOOLEAN
james@204 690 XenNet_TxShutdown(xennet_info_t *xi)
james@204 691 {
james@254 692 PLIST_ENTRY entry;
james@254 693 PNDIS_PACKET packet;
james@535 694 //PMDL mdl;
james@535 695 //ULONG i;
james@256 696 KIRQL OldIrql;
james@206 697
andy@391 698 FUNCTION_ENTER();
james@252 699
james@603 700 KeAcquireSpinLock(&xi->tx_lock, &OldIrql);
james@603 701 xi->tx_shutting_down = TRUE;
james@603 702 KeReleaseSpinLock(&xi->tx_lock, OldIrql);
james@603 703
james@603 704 while (xi->tx_outstanding)
james@603 705 {
james@603 706 KdPrint((__DRIVER_NAME " Waiting for %d remaining packets to be sent\n", xi->tx_outstanding));
james@603 707 KeWaitForSingleObject(&xi->tx_idle_event, Executive, KernelMode, FALSE, NULL);
james@603 708 }
james@603 709
james@586 710 KeRemoveQueueDpc(&xi->tx_dpc);
james@821 711 #if (NTDDI_VERSION >= NTDDI_WINXP)
james@586 712 KeFlushQueuedDpcs();
james@821 713 #endif
james@586 714
james@254 715 /* Free packets in tx queue */
james@254 716 entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
james@254 717 while (entry != &xi->tx_waiting_pkt_list)
james@252 718 {
james@254 719 packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
james@254 720 NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_FAILURE);
james@254 721 entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
james@252 722 }
james@780 723
james@780 724 NdisDeleteNPagedLookasideList(&xi->tx_lookaside_list);
james@250 725
andy@391 726 FUNCTION_EXIT();
james@206 727
james@204 728 return TRUE;
james@204 729 }