win-pvdrivers

changeset 780:3da023729e6b

big update to the xennet tx path. Now passes NDISTest (all 'functional' tests at least).
author James Harper <james.harper@bendigoit.com.au>
date Wed Feb 10 10:11:16 2010 +1100 (2010-02-10)
parents fff1d7255040
children 2b2bf47e8672
files xennet/xennet.c xennet/xennet.h xennet/xennet_common.c xennet/xennet_tx.c xenpci/xenpci.h xenpci/xenpci_pdo.c
line diff
     1.1 --- a/xennet/xennet.c	Tue Feb 09 22:18:47 2010 +1100
     1.2 +++ b/xennet/xennet.c	Wed Feb 10 10:11:16 2010 +1100
     1.3 @@ -678,24 +678,8 @@ XenNet_Init(
     1.4      goto err;
     1.5    }
     1.6  
     1.7 -  if (xi->config_sg)
     1.8 +  if (!xi->config_sg)
     1.9    {
    1.10 -    KdPrint((__DRIVER_NAME "     SG Enabled\n"));
    1.11 -    status = NdisMInitializeScatterGatherDma(xi->adapter_handle, TRUE, xi->config_gso);
    1.12 -    if (!NT_SUCCESS(status))
    1.13 -    {
    1.14 -      KdPrint(("NdisMInitializeScatterGatherDma failed (%08x), disabling\n", status));
    1.15 -      xi->config_sg = 0;
    1.16 -    }
    1.17 -  }
    1.18 -  else
    1.19 -  {
    1.20 -    KdPrint((__DRIVER_NAME "     SG Disabled\n"));
    1.21 -    status = NdisMAllocateMapRegisters(xi->adapter_handle, 0, NDIS_DMA_64BITS, 64, PAGE_SIZE);
    1.22 -    if (status != NDIS_STATUS_SUCCESS)
    1.23 -    {
    1.24 -      KdPrint((__DRIVER_NAME "     Cannot allocate Map Registers\n"));
    1.25 -    }
    1.26      /* without SG, GSO can be a maximum of PAGE_SIZE */
    1.27      xi->config_gso = min(xi->config_gso, PAGE_SIZE);
    1.28    }
    1.29 @@ -804,8 +788,6 @@ XenNet_Reset(
    1.30    return NDIS_STATUS_SUCCESS;
    1.31  }
    1.32  
    1.33 -dma_driver_extension_t *dma_driver_extension;
    1.34 -
    1.35  NTSTATUS DDKAPI
    1.36  DriverEntry(
    1.37    PDRIVER_OBJECT DriverObject,
    1.38 @@ -818,11 +800,6 @@ DriverEntry(
    1.39  
    1.40    FUNCTION_ENTER();
    1.41  
    1.42 -  IoAllocateDriverObjectExtension(DriverObject, UlongToPtr(XEN_DMA_DRIVER_EXTENSION_MAGIC), sizeof(dma_driver_extension_t), &dma_driver_extension);  
    1.43 -  dma_driver_extension->need_virtual_address = NULL;
    1.44 -  dma_driver_extension->get_alignment = NULL;
    1.45 -  dma_driver_extension->max_sg_elements = 19; /* header + 18 fragments */
    1.46 -
    1.47    KdPrint((__DRIVER_NAME "     DriverObject = %p, RegistryPath = %p\n", DriverObject, RegistryPath));
    1.48    RtlZeroMemory(&mini_chars, sizeof(mini_chars));
    1.49  
     2.1 --- a/xennet/xennet.h	Tue Feb 09 22:18:47 2010 +1100
     2.2 +++ b/xennet/xennet.h	Wed Feb 10 10:11:16 2010 +1100
     2.3 @@ -175,6 +175,8 @@ SET_NET_ULONG(PVOID ptr, ULONG data)
     2.4  #define MIN_LOOKAHEAD_LENGTH (MAX_IP4_HEADER_LENGTH + MAX_TCP_HEADER_LENGTH)
     2.5  #define MAX_LOOKAHEAD_LENGTH 256
     2.6  
     2.7 +#define LINUX_MAX_SG_ELEMENTS 19
     2.8 +
     2.9  typedef struct
    2.10  {
    2.11    PVOID next;
    2.12 @@ -189,7 +191,7 @@ typedef struct
    2.13  typedef struct
    2.14  {
    2.15    PNDIS_PACKET packet; /* only set on the last packet */
    2.16 -  shared_buffer_t *cb;
    2.17 +  PVOID *cb;
    2.18    grant_ref_t gref;
    2.19  } tx_shadow_t;
    2.20  
    2.21 @@ -200,7 +202,7 @@ typedef struct {
    2.22    shared_buffer_t *curr_pb;
    2.23    PUCHAR first_buffer_virtual;
    2.24    ULONG mdl_count;
    2.25 -  USHORT curr_mdl_offset;
    2.26 +  ULONG curr_mdl_offset;
    2.27    USHORT mss;
    2.28    NDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
    2.29    BOOLEAN csum_blank;
    2.30 @@ -278,10 +280,12 @@ struct xennet_info
    2.31    ULONG tx_outstanding;
    2.32    ULONG tx_id_free;
    2.33    USHORT tx_id_list[NET_TX_RING_SIZE];
    2.34 -  ULONG tx_cb_free;
    2.35 -  ULONG tx_cb_list[TX_COALESCE_BUFFERS];
    2.36 -  shared_buffer_t tx_cbs[TX_COALESCE_BUFFERS];
    2.37 +  //ULONG tx_cb_free;
    2.38 +  //ULONG tx_cb_list[TX_COALESCE_BUFFERS];
    2.39 +  //ULONG tx_cb_size;
    2.40 +  //shared_buffer_t tx_cbs[TX_COALESCE_BUFFERS];
    2.41    KDPC tx_dpc;
    2.42 +  NPAGED_LOOKASIDE_LIST tx_lookaside_list;
    2.43  
    2.44    /* rx_related - protected by rx_lock */
    2.45    KSPIN_LOCK rx_lock;
    2.46 @@ -428,3 +432,32 @@ XenNet_ClearPacketInfo(packet_info_t *pi
    2.47      pi->data_validated = pi->split_required = 0;
    2.48  #endif
    2.49  }
    2.50 +
    2.51 +/* Get some data from the current packet, but don't cross a page boundry. */
    2.52 +static __forceinline ULONG
    2.53 +XenNet_QueryData(packet_info_t *pi, ULONG length)
    2.54 +{
    2.55 +  ULONG offset_in_page;
    2.56 +  
    2.57 +  if (length > MmGetMdlByteCount(pi->curr_buffer) - pi->curr_mdl_offset)
    2.58 +    length = MmGetMdlByteCount(pi->curr_buffer) - pi->curr_mdl_offset;
    2.59 +
    2.60 +  offset_in_page = (MmGetMdlByteOffset(pi->curr_buffer) + pi->curr_mdl_offset) & (PAGE_SIZE - 1);
    2.61 +  if (offset_in_page + length > PAGE_SIZE)
    2.62 +    length = PAGE_SIZE - offset_in_page;
    2.63 +  
    2.64 +  return length;
    2.65 +}
    2.66 +
    2.67 +/* Move the pointers forward by the given amount. No error checking is done.  */
    2.68 +static __forceinline VOID
    2.69 +XenNet_EatData(packet_info_t *pi, ULONG length)
    2.70 +{
    2.71 +  pi->curr_mdl_offset += length;
    2.72 +  if (pi->curr_mdl_offset >= MmGetMdlByteCount(pi->curr_buffer))
    2.73 +  {
    2.74 +    pi->curr_mdl_offset -= MmGetMdlByteCount(pi->curr_buffer);
    2.75 +    NdisGetNextBuffer(pi->curr_buffer, &pi->curr_buffer);
    2.76 +  }
    2.77 +}
    2.78 +
     3.1 --- a/xennet/xennet_common.c	Tue Feb 09 22:18:47 2010 +1100
     3.2 +++ b/xennet/xennet_common.c	Wed Feb 10 10:11:16 2010 +1100
     3.3 @@ -20,10 +20,7 @@ Foundation, Inc., 51 Franklin Street, Fi
     3.4  
     3.5  #include "xennet.h"
     3.6  
     3.7 -/*
     3.8 -Increase the header to a certain size
     3.9 -*/
    3.10 -
    3.11 +/* Increase the header to a certain size */
    3.12  BOOLEAN
    3.13  XenNet_BuildHeader(packet_info_t *pi, PUCHAR header, ULONG new_header_size)
    3.14  {
     4.1 --- a/xennet/xennet_tx.c	Tue Feb 09 22:18:47 2010 +1100
     4.2 +++ b/xennet/xennet_tx.c	Wed Feb 10 10:11:16 2010 +1100
     4.3 @@ -39,37 +39,28 @@ put_id_on_freelist(struct xennet_info *x
     4.4    xi->tx_id_free++;
     4.5  }
     4.6  
     4.7 -static __inline shared_buffer_t *
     4.8 -get_cb_from_freelist(struct xennet_info *xi)
     4.9 -{
    4.10 -  shared_buffer_t *cb;
    4.11 -  
    4.12 -  //FUNCTION_ENTER();
    4.13 -  if (xi->tx_cb_free == 0)
    4.14 -  {
    4.15 -    //FUNCTION_EXIT();
    4.16 -    return NULL;
    4.17 -  }
    4.18 -  xi->tx_cb_free--;
    4.19 -  cb = &xi->tx_cbs[xi->tx_cb_list[xi->tx_cb_free]];
    4.20 -  //FUNCTION_EXIT();
    4.21 -  return cb;
    4.22 -}
    4.23 -
    4.24 -static __inline VOID
    4.25 -put_cb_on_freelist(struct xennet_info *xi, shared_buffer_t *cb)
    4.26 -{
    4.27 -  //FUNCTION_ENTER();
    4.28 -  
    4.29 -  ASSERT(cb);
    4.30 -  xi->tx_cb_list[xi->tx_cb_free] = cb->id;
    4.31 -  xi->tx_cb_free++;
    4.32 -  //FUNCTION_EXIT();
    4.33 -}
    4.34 -
    4.35  #define SWAP_USHORT(x) (USHORT)((((x & 0xFF) << 8)|((x >> 8) & 0xFF)))
    4.36  
    4.37 -
    4.38 +static __forceinline struct netif_tx_request *
    4.39 +XenNet_PutCbOnRing(struct xennet_info *xi, PVOID coalesce_buf, ULONG length, grant_ref_t gref)
    4.40 +{
    4.41 +  struct netif_tx_request *tx;
    4.42 +  tx = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
    4.43 +  xi->tx.req_prod_pvt++;
    4.44 +  xi->tx_ring_free--;
    4.45 +  tx->id = get_id_from_freelist(xi);
    4.46 +  ASSERT(xi->tx_shadows[tx->id].gref == INVALID_GRANT_REF);
    4.47 +  ASSERT(!xi->tx_shadows[tx->id].cb);
    4.48 +  xi->tx_shadows[tx->id].cb = coalesce_buf;
    4.49 +  tx->gref = xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0, (ULONG)(MmGetPhysicalAddress(coalesce_buf).QuadPart >> PAGE_SHIFT), FALSE, gref);
    4.50 +  xi->tx_shadows[tx->id].gref = tx->gref;
    4.51 +  tx->offset = 0;
    4.52 +  tx->size = (USHORT)length;
    4.53 +  ASSERT(tx->offset + tx->size <= PAGE_SIZE);
    4.54 +  ASSERT(tx->size);
    4.55 +  return tx;
    4.56 +}
    4.57 +  
    4.58  /* Called at DISPATCH_LEVEL with tx_lock held */
    4.59  /*
    4.60   * Send one NDIS_PACKET. This may involve multiple entries on TX ring.
    4.61 @@ -86,33 +77,77 @@ XenNet_HWSendPacket(struct xennet_info *
    4.62    packet_info_t pi;
    4.63    BOOLEAN ndis_lso = FALSE;
    4.64    BOOLEAN xen_gso = FALSE;
    4.65 -  PSCATTER_GATHER_LIST sg = NULL;
    4.66 -  ULONG sg_element = 0;
    4.67 -  ULONG sg_offset = 0;
    4.68 +  ULONG remaining;
    4.69    ULONG parse_result;
    4.70 -  shared_buffer_t *coalesce_buf = NULL;
    4.71 -  ULONG chunks = 0;
    4.72 +  ULONG frags = 0;
    4.73 +  BOOLEAN coalesce_required = FALSE;
    4.74 +  PVOID coalesce_buf;
    4.75 +  ULONG coalesce_remaining = 0;
    4.76 +  grant_ref_t gref;
    4.77 +  ULONG tx_length = 0;
    4.78    
    4.79    //FUNCTION_ENTER();
    4.80 -  
    4.81 +
    4.82 +  gref = xi->vectors.GntTbl_GetRef(xi->vectors.context);
    4.83 +  if (gref == INVALID_GRANT_REF)
    4.84 +  {
    4.85 +    KdPrint((__DRIVER_NAME "     out of grefs\n"));
    4.86 +    return FALSE;
    4.87 +  }
    4.88 +  coalesce_buf = NdisAllocateFromNPagedLookasideList(&xi->tx_lookaside_list);
    4.89 +  if (!coalesce_buf)
    4.90 +  {
    4.91 +    xi->vectors.GntTbl_PutRef(xi->vectors.context, gref);
    4.92 +    KdPrint((__DRIVER_NAME "     out of memory\n"));
    4.93 +    return FALSE;
    4.94 +  }
    4.95    XenNet_ClearPacketInfo(&pi);
    4.96    NdisQueryPacket(packet, NULL, (PUINT)&pi.mdl_count, &pi.first_buffer, (PUINT)&pi.total_length);
    4.97 -
    4.98 -  if (xi->config_sg)
    4.99 -  {
   4.100 -    parse_result = XenNet_ParsePacketHeader(&pi, NULL, 0);
   4.101 -  }
   4.102 -  else
   4.103 +  
   4.104 +  pi.curr_mdl_offset = 0;
   4.105 +  pi.curr_buffer = pi.first_buffer;
   4.106 +  remaining = min(pi.total_length, PAGE_SIZE);
   4.107 +  while (remaining) /* this much gets put in the header */
   4.108    {
   4.109 -    coalesce_buf = get_cb_from_freelist(xi);
   4.110 -    if (!coalesce_buf)
   4.111 +    ULONG length = XenNet_QueryData(&pi, remaining);
   4.112 +    remaining -= length;
   4.113 +    XenNet_EatData(&pi, length);
   4.114 +  }
   4.115 +  frags++;
   4.116 +  if (pi.total_length > PAGE_SIZE) /* these are the frags we care about */
   4.117 +  {
   4.118 +    remaining = pi.total_length - PAGE_SIZE;
   4.119 +    while (remaining)
   4.120      {
   4.121 -      KdPrint((__DRIVER_NAME "     Full on send - no free cb's\n"));
   4.122 -      return FALSE;
   4.123 +      ULONG length = XenNet_QueryData(&pi, PAGE_SIZE);
   4.124 +      if (length != 0)
   4.125 +      {
   4.126 +        frags++;
   4.127 +        if (frags > LINUX_MAX_SG_ELEMENTS)
   4.128 +          break; /* worst case there could be hundreds of fragments - leave the loop now */
   4.129 +      }
   4.130 +      remaining -= length;
   4.131 +      XenNet_EatData(&pi, length);
   4.132      }
   4.133 -    parse_result = XenNet_ParsePacketHeader(&pi, coalesce_buf->virtual, pi.total_length);
   4.134 +  }
   4.135 +  if (frags > LINUX_MAX_SG_ELEMENTS)
   4.136 +  {
   4.137 +    frags = LINUX_MAX_SG_ELEMENTS;
   4.138 +    coalesce_required = TRUE;
   4.139 +  }
   4.140 +
   4.141 +  /* if we have enough space on the ring then we have enough id's so no need to check for that */
   4.142 +  if (xi->tx_ring_free < frags + 1)
   4.143 +  {
   4.144 +    xi->vectors.GntTbl_PutRef(xi->vectors.context, gref);
   4.145 +    NdisFreeToNPagedLookasideList(&xi->tx_lookaside_list, coalesce_buf);
   4.146 +    //KdPrint((__DRIVER_NAME "     Full on send - ring full\n"));
   4.147 +    return FALSE;
   4.148    }
   4.149    
   4.150 +  parse_result = XenNet_ParsePacketHeader(&pi, coalesce_buf, PAGE_SIZE);
   4.151 +  remaining = pi.total_length - pi.header_length;
   4.152 +
   4.153    if (NDIS_GET_PACKET_PROTOCOL_TYPE(packet) == NDIS_PROTOCOL_ID_TCP_IP)
   4.154    {
   4.155      csum_info = (PNDIS_TCP_IP_CHECKSUM_PACKET_INFO)&NDIS_PER_PACKET_INFO_FROM_PACKET(
   4.156 @@ -159,7 +194,7 @@ XenNet_HWSendPacket(struct xennet_info *
   4.157        //return TRUE;
   4.158      }
   4.159    }
   4.160 -    
   4.161 +  
   4.162    mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo));
   4.163  
   4.164    if (mss && parse_result == PARSE_OK)
   4.165 @@ -178,35 +213,6 @@ XenNet_HWSendPacket(struct xennet_info *
   4.166      }
   4.167    }
   4.168  
   4.169 -  if (xi->config_sg)
   4.170 -  {
   4.171 -    sg = (PSCATTER_GATHER_LIST)NDIS_PER_PACKET_INFO_FROM_PACKET(packet, ScatterGatherListPacketInfo);
   4.172 -    ASSERT(sg != NULL);
   4.173 -
   4.174 -    if (sg->NumberOfElements > 19)
   4.175 -    {
   4.176 -      KdPrint((__DRIVER_NAME "     sg->NumberOfElements = %d\n", sg->NumberOfElements));
   4.177 -      NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_SUCCESS);
   4.178 -      return TRUE; // we'll pretend we sent the packet here for now...
   4.179 -    }
   4.180 -    if (sg->NumberOfElements + !!ndis_lso > xi->tx_ring_free)
   4.181 -    {
   4.182 -      //KdPrint((__DRIVER_NAME "     Full on send - required = %d, available = %d\n", sg->NumberOfElements + !!ndis_lso, xi->tx_ring_free));
   4.183 -      //FUNCTION_EXIT();
   4.184 -      return FALSE;
   4.185 -    }
   4.186 -
   4.187 -    if (ndis_lso || (pi.header_length && pi.header_length > sg->Elements[sg_element].Length && pi.header == pi.header_data))
   4.188 -    {
   4.189 -      coalesce_buf = get_cb_from_freelist(xi);
   4.190 -      if (!coalesce_buf)
   4.191 -      {
   4.192 -        KdPrint((__DRIVER_NAME "     Full on send - no free cb's\n"));
   4.193 -        return FALSE;
   4.194 -      }
   4.195 -    }
   4.196 -  }
   4.197 -  
   4.198    if (ndis_lso)
   4.199    {    
   4.200      flags |= NETTXF_csum_blank | NETTXF_data_validated; /* these may be implied but not specified when lso is used*/
   4.201 @@ -227,73 +233,17 @@ XenNet_HWSendPacket(struct xennet_info *
   4.202  */
   4.203  
   4.204    /* (A) */
   4.205 -  tx0 = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
   4.206 -  xi->tx.req_prod_pvt++;
   4.207 -  chunks++;
   4.208 -  xi->tx_ring_free--;
   4.209 -  tx0->id = get_id_from_freelist(xi);
   4.210 -  ASSERT(xi->tx_shadows[tx0->id].gref == INVALID_GRANT_REF);
   4.211 -// if we coalesced the header then we want to put that on first, otherwise we put on the first sg element
   4.212 -  if (coalesce_buf)
   4.213 +  tx0 = XenNet_PutCbOnRing(xi, coalesce_buf, pi.header_length, gref);
   4.214 +  ASSERT(tx0); /* this will never happen */
   4.215 +  tx0->flags = flags;
   4.216 +  tx_length += pi.header_length;
   4.217 +
   4.218 +  /* even though we haven't reported that we are capable of it, LSO demands that we calculate the IP Header checksum */
   4.219 +  if (ndis_lso)
   4.220    {
   4.221 -    ULONG remaining = pi.header_length;
   4.222 -    memcpy(coalesce_buf->virtual, pi.header, pi.header_length);
   4.223 -    /* even though we haven't reported that we are capable of it, LSO demands that we calculate the IP Header checksum */
   4.224 -    if (ndis_lso)
   4.225 -    {
   4.226 -      XenNet_SumIpHeader(coalesce_buf->virtual, pi.ip4_header_length);
   4.227 -    }
   4.228 -    ASSERT(!xi->tx_shadows[tx0->id].cb);
   4.229 -    xi->tx_shadows[tx0->id].cb = coalesce_buf;
   4.230 -    tx0->gref = coalesce_buf->gref;
   4.231 -    tx0->offset = coalesce_buf->offset;
   4.232 -    tx0->size = (USHORT)pi.header_length;
   4.233 -    ASSERT(tx0->offset + tx0->size <= PAGE_SIZE);
   4.234 -    ASSERT(tx0->size);
   4.235 -    if (xi->config_sg)
   4.236 -    {
   4.237 -      /* TODO: if the next buffer contains only a small amount of data then put it on too */
   4.238 -      while (remaining)
   4.239 -      {
   4.240 -        if (sg->Elements[sg_element].Length <= remaining)
   4.241 -        {
   4.242 -          remaining -= sg->Elements[sg_element].Length;
   4.243 -          sg_element++;
   4.244 -        }
   4.245 -        else
   4.246 -        {
   4.247 -          sg_offset = remaining;
   4.248 -          remaining = 0;
   4.249 -        }
   4.250 -      }
   4.251 -    }
   4.252 +    XenNet_SumIpHeader(coalesce_buf, pi.ip4_header_length);
   4.253    }
   4.254 -  else
   4.255 -  {
   4.256 -    ASSERT(xi->config_sg);
   4.257 -    tx0->gref = xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
   4.258 -      (ULONG)(sg->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF);
   4.259 -    ASSERT(tx0->gref != INVALID_GRANT_REF);
   4.260 -    xi->tx_shadows[tx0->id].gref = tx0->gref;
   4.261 -    tx0->offset = (USHORT)sg->Elements[sg_element].Address.LowPart & (PAGE_SIZE - 1);
   4.262 -    tx0->size = (USHORT)min(sg->Elements[sg_element].Length, (ULONG)(PAGE_SIZE - tx0->offset));
   4.263 -    if (tx0->offset + tx0->size > PAGE_SIZE)
   4.264 -    {
   4.265 -      KdPrint((__DRIVER_NAME "     offset + size = %d\n", tx0->offset + tx0->size));
   4.266 -    }
   4.267 -    ASSERT(tx0->size);
   4.268 -    if (tx0->size != sg->Elements[sg_element].Length)
   4.269 -    {
   4.270 -      sg_offset = tx0->size;
   4.271 -    }
   4.272 -    else
   4.273 -    {
   4.274 -      sg_element++;
   4.275 -    }
   4.276 -  }
   4.277 -  tx0->flags = flags;
   4.278    txN = tx0;
   4.279 -  ASSERT(txN->gref != INVALID_GRANT_REF);
   4.280  
   4.281    /* (B) */
   4.282    if (xen_gso)
   4.283 @@ -311,43 +261,104 @@ XenNet_HWSendPacket(struct xennet_info *
   4.284      ei->u.gso.features = 0;
   4.285    }
   4.286  
   4.287 -  if (xi->config_sg)
   4.288 +  ASSERT(xi->config_sg || !remaining);
   4.289 +  
   4.290 +  /* (C) - only if data is remaining */
   4.291 +  coalesce_buf = NULL;
   4.292 +  while (remaining > 0)
   4.293    {
   4.294 -    /* (C) - only if sg otherwise it was all sent on the first buffer */
   4.295 -    while (sg_element < sg->NumberOfElements)
   4.296 +    ULONG length;
   4.297 +    PFN_NUMBER pfn;
   4.298 +    
   4.299 +    ASSERT(pi.curr_buffer);
   4.300 +    if (coalesce_required)
   4.301      {
   4.302 -      txN = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
   4.303 -      xi->tx.req_prod_pvt++;
   4.304 -      chunks++;
   4.305 -      xi->tx_ring_free--;
   4.306 -      txN->id = get_id_from_freelist(xi);
   4.307 -      txN->gref = xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
   4.308 -        (ULONG)((sg->Elements[sg_element].Address.QuadPart + sg_offset) >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF);
   4.309 -      ASSERT(txN->gref != INVALID_GRANT_REF);
   4.310 -      ASSERT(xi->tx_shadows[txN->id].gref == INVALID_GRANT_REF);
   4.311 -      xi->tx_shadows[txN->id].gref = txN->gref;
   4.312 -      txN->offset = (USHORT)(sg->Elements[sg_element].Address.LowPart + sg_offset) & (PAGE_SIZE - 1);
   4.313 -      ASSERT(sg->Elements[sg_element].Length > sg_offset);
   4.314 -      txN->size = (USHORT)min(sg->Elements[sg_element].Length - sg_offset, (ULONG)(PAGE_SIZE - txN->offset));
   4.315 -      if (txN->offset + txN->size > PAGE_SIZE)
   4.316 +      PVOID va;
   4.317 +      if (!coalesce_buf)
   4.318        {
   4.319 -        KdPrint((__DRIVER_NAME "     offset (%d) + size (%d) = %d\n", txN->offset, txN->size, txN->offset + txN->size));
   4.320 +        gref = xi->vectors.GntTbl_GetRef(xi->vectors.context);
   4.321 +        if (gref == INVALID_GRANT_REF)
   4.322 +        {
   4.323 +          KdPrint((__DRIVER_NAME "     out of grefs - partial send\n"));
   4.324 +          break;
   4.325 +        }
   4.326 +        coalesce_buf = NdisAllocateFromNPagedLookasideList(&xi->tx_lookaside_list);
   4.327 +        if (!coalesce_buf)
   4.328 +        {
   4.329 +          xi->vectors.GntTbl_PutRef(xi->vectors.context, gref);
   4.330 +          KdPrint((__DRIVER_NAME "     out of memory - partial send\n"));
   4.331 +          break;
   4.332 +        }
   4.333 +        coalesce_remaining = min(PAGE_SIZE, remaining);
   4.334        }
   4.335 -      ASSERT(txN->offset + txN->size <= PAGE_SIZE);
   4.336 -      ASSERT(txN->size);
   4.337 -      tx0->size = tx0->size + txN->size;
   4.338 -      txN->flags = NETTXF_more_data;
   4.339 -      ASSERT(txN->gref != INVALID_GRANT_REF);
   4.340 -      if (txN->size != sg->Elements[sg_element].Length - sg_offset)
   4.341 +      length = XenNet_QueryData(&pi, coalesce_remaining);
   4.342 +      va = NdisBufferVirtualAddressSafe(pi.curr_buffer, LowPagePriority);
   4.343 +      if (!va)
   4.344        {
   4.345 -        sg_offset += txN->size;
   4.346 +        KdPrint((__DRIVER_NAME "     failed to map buffer va - partial send\n"));
   4.347 +        coalesce_remaining = 0;
   4.348 +        remaining -= min(PAGE_SIZE, remaining);
   4.349 +        NdisFreeToNPagedLookasideList(&xi->tx_lookaside_list, coalesce_buf);
   4.350        }
   4.351        else
   4.352        {
   4.353 -        sg_element++;
   4.354 -        sg_offset = 0;
   4.355 +        memcpy((PUCHAR)coalesce_buf + min(PAGE_SIZE, remaining) - coalesce_remaining, (PUCHAR)va + pi.curr_mdl_offset, length);
   4.356 +        coalesce_remaining -= length;
   4.357        }
   4.358      }
   4.359 +    else
   4.360 +    {
   4.361 +      length = XenNet_QueryData(&pi, PAGE_SIZE);
   4.362 +    }
   4.363 +    if (!length || coalesce_remaining) /* sometimes there are zero length buffers... */
   4.364 +    {
   4.365 +      XenNet_EatData(&pi, length); /* do this so we actually move to the next buffer */
   4.366 +      continue;
   4.367 +    }
   4.368 +
   4.369 +    if (coalesce_buf)
   4.370 +    {
   4.371 +      if (remaining)
   4.372 +      {
   4.373 +        txN = XenNet_PutCbOnRing(xi, coalesce_buf, min(PAGE_SIZE, remaining), gref);
   4.374 +        ASSERT(txN);
   4.375 +        coalesce_buf = NULL;
   4.376 +        remaining -= min(PAGE_SIZE, remaining);
   4.377 +        tx_length += min(PAGE_SIZE, remaining);
   4.378 +      }
   4.379 +    }
   4.380 +    else
   4.381 +    {
   4.382 +      ULONG offset;
   4.383 +      
   4.384 +      gref = xi->vectors.GntTbl_GetRef(xi->vectors.context);
   4.385 +      if (gref == INVALID_GRANT_REF)
   4.386 +      {
   4.387 +        KdPrint((__DRIVER_NAME "     out of grefs - partial send\n"));
   4.388 +        break;
   4.389 +      }
   4.390 +      txN = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
   4.391 +      xi->tx.req_prod_pvt++;
   4.392 +      xi->tx_ring_free--;
   4.393 +      txN->id = get_id_from_freelist(xi);
   4.394 +      ASSERT(!xi->tx_shadows[txN->id].cb);
   4.395 +      offset = MmGetMdlByteOffset(pi.curr_buffer) + pi.curr_mdl_offset;
   4.396 +      pfn = MmGetMdlPfnArray(pi.curr_buffer)[offset >> PAGE_SHIFT];
   4.397 +      txN->offset = (USHORT)offset & (PAGE_SIZE - 1);
   4.398 +      txN->gref = xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0, pfn, FALSE, gref);
   4.399 +      ASSERT(xi->tx_shadows[txN->id].gref == INVALID_GRANT_REF);
   4.400 +      xi->tx_shadows[txN->id].gref = txN->gref;
   4.401 +      //ASSERT(sg->Elements[sg_element].Length > sg_offset);
   4.402 +      txN->size = (USHORT)length;
   4.403 +      ASSERT(txN->offset + txN->size <= PAGE_SIZE);
   4.404 +      ASSERT(txN->size);
   4.405 +      ASSERT(txN->gref != INVALID_GRANT_REF);
   4.406 +      remaining -= length;
   4.407 +      tx_length += length;
   4.408 +    }
   4.409 +    tx0->size = tx0->size + txN->size;
   4.410 +    txN->flags = NETTXF_more_data;
   4.411 +    XenNet_EatData(&pi, length);
   4.412    }
   4.413    txN->flags &= ~NETTXF_more_data;
   4.414    ASSERT(tx0->size == pi.total_length);
   4.415 @@ -357,13 +368,9 @@ XenNet_HWSendPacket(struct xennet_info *
   4.416    if (ndis_lso)
   4.417    {
   4.418      //KdPrint((__DRIVER_NAME "     TcpLargeSendPacketInfo = %d\n", pi.tcp_length));
   4.419 -    NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo) = UlongToPtr(pi.tcp_length);
   4.420 +    NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo) = UlongToPtr(tx_length - MAX_ETH_HEADER_LENGTH - pi.ip4_header_length - pi.tcp_header_length);
   4.421    }
   4.422  
   4.423 -  if (chunks > 19)
   4.424 -  {
   4.425 -    KdPrint((__DRIVER_NAME "     chunks = %d\n", chunks));
   4.426 -  }
   4.427    xi->stat_tx_ok++;
   4.428  
   4.429    //NDIS_SET_PACKET_STATUS(packet, NDIS_STATUS_SUCCESS);
   4.430 @@ -393,7 +400,7 @@ XenNet_SendQueuedPackets(struct xennet_i
   4.431      packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
   4.432      if (!XenNet_HWSendPacket(xi, packet))
   4.433      {
   4.434 -      KdPrint((__DRIVER_NAME "     No room for packet\n"));
   4.435 +      //KdPrint((__DRIVER_NAME "     No room for packet\n"));
   4.436        InsertHeadList(&xi->tx_waiting_pkt_list, entry);
   4.437        break;
   4.438      }
   4.439 @@ -459,8 +466,7 @@ XenNet_TxBufferGC(PKDPC dpc, PVOID conte
   4.440        shadow = &xi->tx_shadows[txrsp->id];
   4.441        if (shadow->cb)
   4.442        {
   4.443 -        ASSERT(shadow->gref == INVALID_GRANT_REF);
   4.444 -        put_cb_on_freelist(xi, shadow->cb);
   4.445 +        NdisFreeToNPagedLookasideList(&xi->tx_lookaside_list, shadow->cb);
   4.446          shadow->cb = NULL;
   4.447        }
   4.448        
   4.449 @@ -644,9 +650,7 @@ XenNet_TxResumeEnd(xennet_info_t *xi)
   4.450  BOOLEAN
   4.451  XenNet_TxInit(xennet_info_t *xi)
   4.452  {
   4.453 -  NTSTATUS status;
   4.454 -  USHORT i, j;
   4.455 -  ULONG cb_size;
   4.456 +  USHORT i;
   4.457  
   4.458    KeInitializeSpinLock(&xi->tx_lock);
   4.459    KeInitializeDpc(&xi->tx_dpc, XenNet_TxBufferGC, xi);
   4.460 @@ -660,45 +664,8 @@ XenNet_TxInit(xennet_info_t *xi)
   4.461    xi->tx_outstanding = 0;
   4.462    xi->tx_ring_free = NET_TX_RING_SIZE;
   4.463    
   4.464 -  if (xi->config_sg)
   4.465 -  {
   4.466 -    cb_size = TX_HEADER_BUFFER_SIZE;
   4.467 -  }
   4.468 -  else
   4.469 -  {
   4.470 -    cb_size = PAGE_SIZE;
   4.471 -  }
   4.472 -
   4.473 -  for (i = 0; i < TX_COALESCE_BUFFERS / (PAGE_SIZE / cb_size); i++)
   4.474 -  {
   4.475 -    PVOID virtual;
   4.476 -    grant_ref_t gref;
   4.477 -    
   4.478 -    status = NdisAllocateMemoryWithTag(&virtual, PAGE_SIZE, XENNET_POOL_TAG);
   4.479 -    if (status != STATUS_SUCCESS)
   4.480 -    {
   4.481 -      break;
   4.482 -    }
   4.483 -    gref = (grant_ref_t)xi->vectors.GntTbl_GrantAccess(xi->vectors.context, 0,
   4.484 -      (ULONG)(MmGetPhysicalAddress(virtual).QuadPart >> PAGE_SHIFT), FALSE, INVALID_GRANT_REF);
   4.485 -    if (gref == INVALID_GRANT_REF)
   4.486 -    {
   4.487 -      NdisFreeMemory(virtual, PAGE_SIZE, 0);
   4.488 -      break;
   4.489 -    }
   4.490 -    
   4.491 -    for (j = 0; j < PAGE_SIZE / cb_size; j++)
   4.492 -    {
   4.493 -      USHORT index = (USHORT)(i * (PAGE_SIZE / cb_size) + j);
   4.494 -      xi->tx_cbs[index].id = index;
   4.495 -      xi->tx_cbs[index].virtual = (PUCHAR)virtual + j * cb_size;
   4.496 -      xi->tx_cbs[index].gref = gref;
   4.497 -      xi->tx_cbs[index].offset = (ULONG_PTR)xi->tx_cbs[index].virtual & (PAGE_SIZE - 1);
   4.498 -      put_cb_on_freelist(xi, &xi->tx_cbs[index]);
   4.499 -    }
   4.500 -  }
   4.501 -  if (i == 0)
   4.502 -    KdPrint((__DRIVER_NAME "     Unable to allocate any SharedMemory buffers\n"));
   4.503 +  NdisInitializeNPagedLookasideList(&xi->tx_lookaside_list, NULL, NULL, 0,
   4.504 +    PAGE_SIZE, XENNET_POOL_TAG, 0);
   4.505  
   4.506    xi->tx_id_free = 0;
   4.507    for (i = 0; i < NET_TX_RING_SIZE; i++)
   4.508 @@ -724,7 +691,6 @@ XenNet_TxShutdown(xennet_info_t *xi)
   4.509    //PMDL mdl;
   4.510    //ULONG i;
   4.511    KIRQL OldIrql;
   4.512 -  shared_buffer_t *cb;
   4.513  
   4.514    FUNCTION_ENTER();
   4.515  
   4.516 @@ -749,13 +715,8 @@ XenNet_TxShutdown(xennet_info_t *xi)
   4.517      NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_FAILURE);
   4.518      entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
   4.519    }
   4.520 -  
   4.521 -  while((cb = get_cb_from_freelist(xi)) != NULL)
   4.522 -  {
   4.523 -    /* only free the actual buffers which were aligned on a page boundary */
   4.524 -    if ((PtrToUlong(cb->virtual) & (PAGE_SIZE - 1)) == 0)
   4.525 -      NdisFreeMemory(cb->virtual, PAGE_SIZE, 0);
   4.526 -  }
   4.527 +
   4.528 +  NdisDeleteNPagedLookasideList(&xi->tx_lookaside_list);
   4.529  
   4.530    FUNCTION_EXIT();
   4.531  
     5.1 --- a/xenpci/xenpci.h	Tue Feb 09 22:18:47 2010 +1100
     5.2 +++ b/xenpci/xenpci.h	Wed Feb 10 10:11:16 2010 +1100
     5.3 @@ -474,9 +474,4 @@ GntTbl_PutRef(PVOID Context, grant_ref_t
     5.4  grant_ref_t
     5.5  GntTbl_GetRef(PVOID Context);
     5.6  
     5.7 -TRANSLATE_BUS_ADDRESS XenPci_BIS_TranslateBusAddress;
     5.8 -GET_DMA_ADAPTER XenPci_BIS_GetDmaAdapter;
     5.9 -GET_SET_DEVICE_DATA XenPci_BIS_SetBusData;
    5.10 -GET_SET_DEVICE_DATA XenPci_BIS_GetBusData;
    5.11 -
    5.12  #endif
     6.1 --- a/xenpci/xenpci_pdo.c	Tue Feb 09 22:18:47 2010 +1100
     6.2 +++ b/xenpci/xenpci_pdo.c	Wed Feb 10 10:11:16 2010 +1100
     6.3 @@ -1337,17 +1337,6 @@ XenPci_EvtChildListCreateDevice(WDFCHILD
     6.4      KdPrint((__DRIVER_NAME "     WdfFdoQueryForInterface failed - %08x\n", status));
     6.5      return status;
     6.6    }
     6.7 -#if 0
     6.8 -  bus_interface.Size = sizeof(BUS_INTERFACE_STANDARD);
     6.9 -  bus_interface.Version = 1; //BUS_INTERFACE_STANDARD_VERSION;
    6.10 -  bus_interface.Context = xppdd;
    6.11 -  bus_interface.InterfaceReference = WdfDeviceInterfaceReferenceNoOp;
    6.12 -  bus_interface.InterfaceDereference = WdfDeviceInterfaceDereferenceNoOp;
    6.13 -  bus_interface.TranslateBusAddress = XenPci_BIS_TranslateBusAddress;
    6.14 -  bus_interface.GetDmaAdapter = XenPci_BIS_GetDmaAdapter;
    6.15 -  bus_interface.SetBusData = XenPci_BIS_SetBusData;
    6.16 -  bus_interface.GetBusData = XenPci_BIS_GetBusData;
    6.17 -#endif
    6.18    WDF_QUERY_INTERFACE_CONFIG_INIT(&interface_config, (PINTERFACE)&bus_interface, &GUID_BUS_INTERFACE_STANDARD, NULL);
    6.19    status = WdfDeviceAddQueryInterface(child_device, &interface_config);
    6.20    if (!NT_SUCCESS(status))