win-pvdrivers

changeset 229:a9a5b54bee92

refactor code further to simplify flow and break up into easier-to-understand chunks.
TX ring full not yet handled properly.
author Andy Grover <andy.grover@oracle.com>
date Wed Mar 26 18:37:06 2008 -0700 (2008-03-26)
parents 90fcd61a9cbb
children 93f3ccb775b6
files xennet/xennet_tx.c
line diff
     1.1 --- a/xennet/xennet_tx.c	Thu Mar 20 17:51:37 2008 -0700
     1.2 +++ b/xennet/xennet_tx.c	Wed Mar 26 18:37:06 2008 -0700
     1.3 @@ -20,11 +20,19 @@ Foundation, Inc., 51 Franklin Street, Fi
     1.4  
     1.5  #include "xennet.h"
     1.6  
     1.7 +#define FREELIST_ID_ERROR 0xFFFF
     1.8 +
     1.9 +#ifdef XEN_PROFILE
    1.10 +#define PC_INC(var) var++
    1.11 +#else
    1.12 +#define PC_INC(var)
    1.13 +#endif
    1.14 +
    1.15  static USHORT
    1.16  get_id_from_freelist(struct xennet_info *xi)
    1.17  {
    1.18    if (xi->tx_id_free - xi->tx_no_id_free == 0)
    1.19 -    return 0xFFFF;
    1.20 +    return FREELIST_ID_ERROR;
    1.21    xi->tx_id_free--;
    1.22    return xi->tx_id_list[xi->tx_id_free];
    1.23  }
    1.24 @@ -33,7 +41,7 @@ static USHORT
    1.25  get_no_id_from_freelist(struct xennet_info *xi)
    1.26  {
    1.27    if (xi->tx_id_free - xi->tx_no_id_free == 0)
    1.28 -    return 0xFFFF;
    1.29 +    return FREELIST_ID_ERROR;
    1.30    xi->tx_no_id_free--;
    1.31    return 0;
    1.32  }
    1.33 @@ -108,186 +116,147 @@ XenNet_SumHeader(
    1.34    ushorts[5] = SWAP_USHORT(~csum);
    1.35  }
    1.36  
    1.37 +/* Place a buffer on tx ring. */
    1.38 +static struct netif_tx_request*
    1.39 +XenNet_PutOnTxRing(struct xennet_info *xi, ULONGLONG addr, size_t len, uint16_t flags)
    1.40 +{
    1.41 +  struct netif_tx_request *tx;
    1.42 +  unsigned short id;
    1.43 +  PFN_NUMBER pfn = (PFN_NUMBER)(addr >> PAGE_SHIFT);
    1.44 +  ULONG offset = BYTE_OFFSET(addr);
    1.45 +
    1.46 +  id = get_id_from_freelist(xi);
    1.47 +  /* TODO: check id against FREELIST_ID_ERROR */
    1.48 +  ASSERT(xi->tx_pkts[id] == NULL);
    1.49 +  tx = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
    1.50 +
    1.51 +  tx->gref = get_gref_from_freelist(xi);
    1.52 +  ASSERT(tx->gref != 0);
    1.53 +  ASSERT(xi->tx_grefs[id] == 0);
    1.54 +  xi->tx_grefs[id] = tx->gref;
    1.55 +
    1.56 +  xi->XenInterface.GntTbl_GrantAccess(
    1.57 +    xi->XenInterface.InterfaceHeader.Context, 0,
    1.58 +    pfn, FALSE, tx->gref);
    1.59 +  tx->id = id;
    1.60 +  tx->offset = (uint16_t)offset;
    1.61 +  tx->size = (uint16_t)len;
    1.62 +  tx->flags = flags;
    1.63 +  PC_INC(ProfCount_TxPacketsTotal);
    1.64 +
    1.65 +  return tx;
    1.66 +}
    1.67 +
    1.68 +
    1.69  /* Called at DISPATCH_LEVEL with tx_lock held */
    1.70  
    1.71 +/*
    1.72 + * Send one NDIS_PACKET. This may involve multiple entries on TX ring.
    1.73 + */
    1.74  static BOOLEAN
    1.75  XenNet_HWSendPacket(struct xennet_info *xi, PNDIS_PACKET packet)
    1.76  {
    1.77 -  struct netif_tx_request *tx;
    1.78 +  struct netif_tx_request *tx = NULL;
    1.79    struct netif_extra_info *ei;
    1.80 -  unsigned short id;
    1.81 -#if defined(XEN_PROFILE)
    1.82 -  LARGE_INTEGER tsc, dummy;
    1.83 -#endif
    1.84    PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
    1.85 -  
    1.86 -  ULONG i = 0;
    1.87    PSCATTER_GATHER_LIST sg_list;
    1.88    UINT total_packet_length;
    1.89 -  USHORT remaining = 0;
    1.90 -  USHORT offset = 0;
    1.91 -  USHORT length;
    1.92 -  ULONGLONG curr_addr = 0;
    1.93    ULONG sg_num = 0;
    1.94 -  ULONG pfn;
    1.95 -  ULONG mss = 0;
    1.96 +  ULONG mss; // 0 if not using large send
    1.97    PMDL first_buffer;
    1.98    int cycles = 0;
    1.99 +  ULONGLONG sg_elem_addr;
   1.100 +  ULONG sg_elem_len;
   1.101 +  ULONG sg_elem_pages;
   1.102 +  ULONG sg_elem_page;
   1.103 +  ULONG chunk_len;
   1.104 +  uint16_t flags;
   1.105 +#if defined(XEN_PROFILE)
   1.106 +  LARGE_INTEGER tsc, dummy;
   1.107  
   1.108 -#if defined(XEN_PROFILE)
   1.109    tsc = KeQueryPerformanceCounter(&dummy);
   1.110  #endif
   1.111  
   1.112    NdisQueryPacket(packet, NULL, NULL, &first_buffer, &total_packet_length);
   1.113 +
   1.114 +  flags = NETTXF_more_data;
   1.115 +  mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo));
   1.116 +  if (mss > 0)
   1.117 +  {
   1.118 +    flags |= NETTXF_extra_info | NETTXF_csum_blank | NETTXF_data_validated;
   1.119 +    XenNet_SumHeader(first_buffer);
   1.120 +    PC_INC(ProfCount_TxPacketsLargeOffload);
   1.121 +  }
   1.122 +
   1.123    sg_list = NDIS_PER_PACKET_INFO_FROM_PACKET(packet, ScatterGatherListPacketInfo);
   1.124  
   1.125 -  while (sg_num < sg_list->NumberOfElements || remaining || (i == 1 && mss))
   1.126 +  /*
   1.127 +   * See io/netif.h. Must put (A) 1st packet, then (B) optional extra_info, then
   1.128 +   * (C) rest of packets on the ring.
   1.129 +   */
   1.130 +  /* (A) */
   1.131 +  tx = XenNet_PutOnTxRing(xi, sg_list->Elements[0].Address.QuadPart,
   1.132 +    sg_list->Elements[0].Length, flags);
   1.133 +  xi->tx.req_prod_pvt++;
   1.134 +
   1.135 +  /* (B) */
   1.136 +  if (mss > 0)
   1.137 +  {
   1.138 +    get_no_id_from_freelist(xi);
   1.139 +    ei = (struct netif_extra_info *)RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
   1.140 +    ei->type = XEN_NETIF_EXTRA_TYPE_GSO;
   1.141 +    ei->flags = NETTXF_more_data;
   1.142 +    ei->u.gso.size = (USHORT) mss;
   1.143 +    ei->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
   1.144 +    ei->u.gso.pad = 0;
   1.145 +    ei->u.gso.features = 0;
   1.146 +
   1.147 +    xi->tx.req_prod_pvt++;
   1.148 +  }
   1.149 +
   1.150 +  /* (C) */
   1.151 +  for (sg_num = 1; sg_num < sg_list->NumberOfElements; sg_num++)
   1.152    {
   1.153      //KdPrint((__DRIVER_NAME "     i = %d\n", i));
   1.154      ASSERT(cycles++ < 256);
   1.155 -    if (i == 1 && mss)
   1.156 -    {
   1.157 -      //KdPrint((__DRIVER_NAME "     Start of loop - Large Send...\n"));
   1.158 -      length = 0;
   1.159 -    }
   1.160 -    else if (remaining == 0)
   1.161 +
   1.162 +    sg_elem_addr = sg_list->Elements[sg_num].Address.QuadPart;
   1.163 +    sg_elem_len = sg_list->Elements[sg_num].Length;
   1.164 +    sg_elem_pages = ADDRESS_AND_SIZE_TO_SPAN_PAGES(sg_elem_addr, sg_elem_len);
   1.165 +    for (sg_elem_page = 0; sg_elem_page < sg_elem_pages; sg_elem_page++)
   1.166      {
   1.167 -      mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo));
   1.168 -      if (total_packet_length <= mss)
   1.169 -      {
   1.170 -        mss = 0;
   1.171 -      }
   1.172 -      //if (mss)
   1.173 -      //KdPrint((__DRIVER_NAME "     Start of loop - First Frag in sg...\n"));
   1.174 -      curr_addr = sg_list->Elements[sg_num].Address.QuadPart;
   1.175 -      offset = (USHORT)(sg_list->Elements[sg_num].Address.QuadPart & (PAGE_SIZE - 1));
   1.176 -      remaining = (USHORT)sg_list->Elements[sg_num].Length;
   1.177 -      length = min(remaining, PAGE_SIZE - offset);
   1.178 -      //if (mss)
   1.179 -      //KdPrint((__DRIVER_NAME "     sg entry %d - start = %08x, length = %d\n", sg_num, (ULONG)curr_addr, length));
   1.180 -      sg_num++;
   1.181 -    }
   1.182 -    else
   1.183 -    {
   1.184 -      //if (mss)
   1.185 -      //KdPrint((__DRIVER_NAME "     Start of loop - Subsequent Frag in sg...\n"));
   1.186 -      offset = 0;
   1.187 -      length = min(remaining, PAGE_SIZE);
   1.188 -      //if (mss)
   1.189 -      //KdPrint((__DRIVER_NAME "     sg entry %d - start = %08x, length = %d\n", sg_num, (ULONG)curr_addr, length));
   1.190 -    }
   1.191 -    remaining = remaining - length;
   1.192 -    pfn = (ULONG)(curr_addr >> PAGE_SHIFT);
   1.193 -    curr_addr += length;
   1.194 +      chunk_len = min(sg_elem_len, PAGE_SIZE - BYTE_OFFSET(sg_elem_addr));
   1.195  
   1.196 -    if (i++ < *(ULONG *)&packet->MiniportReservedEx)
   1.197 -      continue;
   1.198 -    if (length > 0)
   1.199 -    {
   1.200 -      id = get_id_from_freelist(xi);
   1.201 -      if (id == 0xFFFF)
   1.202 +      flags = NETTXF_more_data;
   1.203 +      csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
   1.204 +        packet, TcpIpChecksumPacketInfo);
   1.205 +      if (csum_info->Transmit.NdisPacketTcpChecksum
   1.206 +        || csum_info->Transmit.NdisPacketUdpChecksum)
   1.207        {
   1.208 -        KdPrint((__DRIVER_NAME "     Out of space...\n"));
   1.209 -        /* whups, out of space on the ring. requeue and get out */
   1.210 -        /* TODO: preallocate all ids so we can exit cleanly here? */
   1.211 -        return FALSE;
   1.212 -        //InsertHeadList(&xi->tx_waiting_pkt_list, entry);
   1.213 -        //break;
   1.214 -      }
   1.215 -      ASSERT(xi->tx_pkts[id] == NULL);
   1.216 -      (*(ULONG *)&packet->MiniportReservedEx)++;
   1.217 -      tx = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
   1.218 -
   1.219 -      tx->gref = get_gref_from_freelist(xi);
   1.220 -      ASSERT(tx->gref != 0);
   1.221 -      ASSERT(xi->tx_grefs[id] == 0);
   1.222 -      xi->tx_grefs[id] = tx->gref;
   1.223 -
   1.224 -      xi->XenInterface.GntTbl_GrantAccess(
   1.225 -        xi->XenInterface.InterfaceHeader.Context, 0,
   1.226 -        pfn, FALSE, tx->gref);
   1.227 -      tx->id = id;
   1.228 -      tx->offset = offset;
   1.229 -      tx->flags = 0;
   1.230 -      if (i != 1)
   1.231 -      {
   1.232 -        //if (mss)
   1.233 -        //KdPrint((__DRIVER_NAME "     Subsequent Frag in packet...\n"));
   1.234 -        tx->size = length;
   1.235 -      }
   1.236 -      else // we have already incremented i!!!
   1.237 -      {
   1.238 -        //if (mss)
   1.239 -        //KdPrint((__DRIVER_NAME "     First Frag in packet...\n"));
   1.240 -        tx->size = (USHORT)total_packet_length;
   1.241 -#if defined(XEN_PROFILE)
   1.242 -        ProfCount_TxPacketsTotal++;
   1.243 -#endif
   1.244 -        csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
   1.245 -          packet, TcpIpChecksumPacketInfo);
   1.246 -        if (csum_info->Transmit.NdisPacketTcpChecksum
   1.247 -          || csum_info->Transmit.NdisPacketUdpChecksum)
   1.248 -        {
   1.249 -          tx->flags |= NETTXF_csum_blank|NETTXF_data_validated;
   1.250 -#if defined(XEN_PROFILE)
   1.251 -          ProfCount_TxPacketsCsumOffload++;
   1.252 -#endif
   1.253 -        }
   1.254 -
   1.255 -        if (mss)
   1.256 -        {
   1.257 -          XenNet_SumHeader(first_buffer);
   1.258 -          //KdPrint((__DRIVER_NAME "     Large Send Offload - mss = %d, length = %d\n", mss, total_packet_length));
   1.259 -          tx->flags |= NETTXF_extra_info|NETTXF_csum_blank|NETTXF_data_validated;
   1.260 -#if defined(XEN_PROFILE)
   1.261 -          ProfCount_TxPacketsLargeOffload++;
   1.262 -#endif
   1.263 -        }
   1.264 +        flags |= NETTXF_csum_blank | NETTXF_data_validated;
   1.265 +        PC_INC(ProfCount_TxPacketsCsumOffload);
   1.266        }
   1.267  
   1.268 -      if (sg_num == sg_list->NumberOfElements && remaining == 0)
   1.269 -      {
   1.270 -        //if (mss)
   1.271 -        //KdPrint((__DRIVER_NAME "     No more frags\n"));
   1.272 -        xi->tx_pkts[id] = packet; /* only set the packet on the last buffer */
   1.273 -      }
   1.274 -      else
   1.275 -      {
   1.276 -        //if (mss)
   1.277 -        //KdPrint((__DRIVER_NAME "     More frags\n"));
   1.278 -        tx->flags |= NETTXF_more_data;
   1.279 -      }
   1.280 +      tx = XenNet_PutOnTxRing(xi, sg_elem_addr, chunk_len, flags);
   1.281 +
   1.282 +      sg_elem_addr += chunk_len;
   1.283 +      sg_elem_len -= chunk_len;
   1.284 +
   1.285 +      xi->tx.req_prod_pvt++;
   1.286      }
   1.287 -    else
   1.288 -    {
   1.289 -      id = get_no_id_from_freelist(xi);
   1.290 -      if (id == 0xFFFF)
   1.291 -      {
   1.292 -        KdPrint((__DRIVER_NAME "     Out of space...\n"));
   1.293 -        /* whups, out of space on the ring. requeue and get out */
   1.294 -        /* TODO: preallocate all ids so we can exit cleanly here? */
   1.295 -        return FALSE;
   1.296 -        //InsertHeadList(&xi->tx_waiting_pkt_list, entry);
   1.297 -        //break;
   1.298 -      }
   1.299 -      //if (mss)
   1.300 -      //KdPrint((__DRIVER_NAME "     Extra Info...\n"));
   1.301 -      (*(ULONG *)&packet->MiniportReservedEx)++;
   1.302 -      ei = (struct netif_extra_info *)RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
   1.303 -      ei->type = XEN_NETIF_EXTRA_TYPE_GSO;
   1.304 -      ei->flags = 0;
   1.305 -      ei->u.gso.size = (USHORT)mss;
   1.306 -      ei->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
   1.307 -      ei->u.gso.pad = 0;
   1.308 -      ei->u.gso.features = 0;
   1.309 -    }
   1.310 -    xi->tx.req_prod_pvt++;
   1.311    }
   1.312  
   1.313 +  /* only set the packet on the last buffer, clear more_data */
   1.314 +  ASSERT(tx);
   1.315 +  ASSERT(tx->id);
   1.316 +  xi->tx_pkts[tx->id] = packet;
   1.317 +  tx->flags &= ~NETTXF_more_data;
   1.318 +
   1.319    return TRUE;
   1.320  }
   1.321  
   1.322  /* Called at DISPATCH_LEVEL with tx_lock held */
   1.323 +/* TODO: calc tx ring slots beforehand and abort if not enough available */
   1.324  
   1.325  static VOID
   1.326  XenNet_SendQueuedPackets(struct xennet_info *xi)