win-pvdrivers

changeset 254:2d25f964e1d1 0.8.9

bug fixes - TX bug appears to be fixed. Improved memory management in the TX path.
author James Harper <james.harper@bendigoit.com.au>
date Sat Apr 26 16:50:31 2008 +1000 (2008-04-26)
parents 58ce01887603
children 77c9e91f543e
files common.inc xennet/xennet.h xennet/xennet_common.c xennet/xennet_rx.c xennet/xennet_tx.c xenpci/evtchn.c xenpci/gnttbl.c xenpci/xenpci.c
line diff
     1.1 --- a/common.inc	Sat Apr 19 00:36:34 2008 +1000
     1.2 +++ b/common.inc	Sat Apr 26 16:50:31 2008 +1000
     1.3 @@ -1,4 +1,4 @@
     1.4 -VERSION=0.8.8.65
     1.5 +VERSION=0.8.9.1
     1.6  TARGETPATH=..\Target\$(DDK_TARGET_OS)
     1.7  KMDF_VERSION=1
     1.8  !IF $(_NT_TOOLS_VERSION) > 0x700
     2.1 --- a/xennet/xennet.h	Sat Apr 19 00:36:34 2008 +1000
     2.2 +++ b/xennet/xennet.h	Sat Apr 26 16:50:31 2008 +1000
     2.3 @@ -176,12 +176,8 @@ struct xennet_info
     2.4    ULONG tx_id_free;
     2.5    ULONG tx_no_id_used;
     2.6    USHORT tx_id_list[NET_TX_RING_SIZE];
     2.7 -  grant_ref_t tx_gref_list[NET_TX_RING_SIZE];
     2.8    PNDIS_PACKET tx_pkts[NET_TX_RING_SIZE];
     2.9    PNDIS_BUFFER tx_mdls[NET_TX_RING_SIZE];
    2.10 -  grant_ref_t tx_grefs[NET_TX_RING_SIZE];
    2.11 -  ULONG tx_gref_free;
    2.12 -  ULONG tx_gref_free_lowest;
    2.13    PMDL tx_page_list[NET_RX_RING_SIZE];
    2.14    ULONG tx_page_free;
    2.15    ULONG tx_page_free_lowest;
    2.16 @@ -208,7 +204,6 @@ struct xennet_info
    2.17  
    2.18    /* how many packets are in the net stack atm */
    2.19    LONG rx_outstanding;
    2.20 -  LONG tx_outstanding;
    2.21  
    2.22    /* config vars from registry */
    2.23    ULONG config_sg;
     3.1 --- a/xennet/xennet_common.c	Sat Apr 19 00:36:34 2008 +1000
     3.2 +++ b/xennet/xennet_common.c	Sat Apr 26 16:50:31 2008 +1000
     3.3 @@ -100,6 +100,9 @@ XenNet_SumIpHeader(
     3.4    ULONG csum = 0;
     3.5    USHORT i;
     3.6  
     3.7 +  ASSERT(ip4_header_length > 12);
     3.8 +  ASSERT(!(ip4_header_length & 1));
     3.9 +
    3.10    header[XN_HDR_SIZE + 10] = 0;
    3.11    header[XN_HDR_SIZE + 11] = 0;
    3.12    for (i = 0; i < ip4_header_length; i += 2)
     4.1 --- a/xennet/xennet_rx.c	Sat Apr 19 00:36:34 2008 +1000
     4.2 +++ b/xennet/xennet_rx.c	Sat Apr 26 16:50:31 2008 +1000
     4.3 @@ -25,16 +25,12 @@ get_page_from_freelist(struct xennet_inf
     4.4  {
     4.5    PMDL mdl;
     4.6  
     4.7 -//  KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n"));
     4.8 -
     4.9    if (xi->rx_page_free == 0)
    4.10    {
    4.11      mdl = AllocatePagesExtra(1, sizeof(grant_ref_t));
    4.12      *(grant_ref_t *)(((UCHAR *)mdl) + MmSizeOfMdl(0, PAGE_SIZE)) = xi->XenInterface.GntTbl_GrantAccess(
    4.13        xi->XenInterface.InterfaceHeader.Context, 0,
    4.14        *MmGetMdlPfnArray(mdl), FALSE, 0);
    4.15 -//    KdPrint(("New Mdl = %p, MmGetMdlVirtualAddress = %p, MmGetSystemAddressForMdlSafe = %p\n",
    4.16 -//      mdl, MmGetMdlVirtualAddress(mdl), MmGetSystemAddressForMdlSafe(mdl, NormalPagePriority)));
    4.17    }
    4.18    else
    4.19    {
    4.20 @@ -42,21 +38,22 @@ get_page_from_freelist(struct xennet_inf
    4.21      if (xi->rx_page_free < xi->rx_page_free_lowest)
    4.22        xi->rx_page_free_lowest = xi->rx_page_free;
    4.23      mdl = xi->rx_page_list[xi->rx_page_free];
    4.24 -//    KdPrint(("Old Mdl = %p, MmGetMdlVirtualAddress = %p, MmGetSystemAddressForMdlSafe = %p\n",
    4.25 -//      mdl, MmGetMdlVirtualAddress(mdl), MmGetSystemAddressForMdlSafe(mdl, NormalPagePriority)));
    4.26    }
    4.27  
    4.28 -//  KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ "\n"));
    4.29 +  return mdl;
    4.30 +}
    4.31  
    4.32 -  return mdl;
    4.33 +static VOID
    4.34 +put_page_on_freelist(struct xennet_info *xi, PMDL mdl)
    4.35 +{
    4.36 +  xi->rx_page_list[xi->rx_page_free] = mdl;
    4.37 +  xi->rx_page_free++;
    4.38  }
    4.39  
    4.40  static VOID
    4.41  free_page_freelist(struct xennet_info *xi)
    4.42  {
    4.43    PMDL mdl;
    4.44 -//  KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n"));
    4.45 -
    4.46    while(xi->rx_page_free != 0)
    4.47    {
    4.48      xi->rx_page_free--;
    4.49 @@ -67,19 +64,6 @@ free_page_freelist(struct xennet_info *x
    4.50    }
    4.51  }
    4.52  
    4.53 -static VOID
    4.54 -put_page_on_freelist(struct xennet_info *xi, PMDL mdl)
    4.55 -{
    4.56 -//  KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n"));
    4.57 -
    4.58 -//  KdPrint(("Mdl = %p\n",  mdl));
    4.59 -
    4.60 -  xi->rx_page_list[xi->rx_page_free] = mdl;
    4.61 -  xi->rx_page_free++;
    4.62 -
    4.63 -//  KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ "\n"));
    4.64 -}
    4.65 -
    4.66  static __inline grant_ref_t
    4.67  get_grant_ref(PMDL mdl)
    4.68  {
    4.69 @@ -363,6 +347,7 @@ XenNet_MakePackets(
    4.70      XenNet_SumPacketData(&xi->rxpi, packets[*packet_count_p]);
    4.71      (*packet_count_p)++;
    4.72    }
    4.73 +
    4.74    ASSERT(xi->rxpi.curr_mdl == xi->rxpi.mdl_count);
    4.75    // TODO: restore psh status to last packet
    4.76    for (i = 0; i < xi->rxpi.mdl_count; i++)
    4.77 @@ -409,7 +394,7 @@ XenNet_RxBufferCheck(struct xennet_info 
    4.78      prod = xi->rx.sring->rsp_prod;
    4.79      KeMemoryBarrier(); /* Ensure we see responses up to 'rp'. */
    4.80  
    4.81 -    for (cons = xi->rx.rsp_cons; cons != prod; cons++)
    4.82 +    for (cons = xi->rx.rsp_cons; cons != prod && packet_count < MAXIMUM_PACKETS_PER_INDICATE; cons++)
    4.83      {
    4.84        ASSERT(cycles++ < 256);
    4.85        id = (USHORT)(cons & (NET_RX_RING_SIZE - 1));
    4.86 @@ -476,7 +461,7 @@ XenNet_RxBufferCheck(struct xennet_info 
    4.87        }
    4.88      }
    4.89      ASSERT(packet_count < NET_RX_RING_SIZE);
    4.90 -    xi->rx.rsp_cons = prod;
    4.91 +    xi->rx.rsp_cons = cons;
    4.92  
    4.93      if (packet_count > 0)
    4.94      {
    4.95 @@ -572,7 +557,8 @@ XenNet_RxTimer(
    4.96  
    4.97    KeAcquireSpinLockAtDpcLevel(&xi->rx_lock);
    4.98  
    4.99 -//  KdPrint((__DRIVER_NAME " --- rx_timer - lowest = %d\n", xi->rx_page_free_lowest));
   4.100 +  KdPrint((__DRIVER_NAME " --- rx_timer - rx_page_free_lowest = %d\n", xi->rx_page_free_lowest));
   4.101 +//  KdPrint((__DRIVER_NAME " --- rx_outstanding = %d, rx_id_free = %d\n", xi->rx_outstanding, xi->rx_id_free));
   4.102  
   4.103    if (xi->rx_page_free_lowest > max(RX_DFL_MIN_TARGET / 4, 16)) // lots of potential for tuning here
   4.104    {
     5.1 --- a/xennet/xennet_tx.c	Sat Apr 19 00:36:34 2008 +1000
     5.2 +++ b/xennet/xennet_tx.c	Sat Apr 26 16:50:31 2008 +1000
     5.3 @@ -38,8 +38,12 @@ static USHORT
     5.4  get_id_from_freelist(struct xennet_info *xi)
     5.5  {
     5.6    if (xi->tx_id_free - xi->tx_no_id_used == 0)
     5.7 +  {
     5.8 +    KdPrint((__DRIVER_NAME "     Out of id's\n"));    
     5.9      return FREELIST_ID_ERROR;
    5.10 +  }
    5.11    xi->tx_id_free--;
    5.12 +
    5.13    return xi->tx_id_list[xi->tx_id_free];
    5.14  }
    5.15  
    5.16 @@ -47,7 +51,10 @@ static USHORT
    5.17  get_no_id_from_freelist(struct xennet_info *xi)
    5.18  {
    5.19    if (xi->tx_id_free - xi->tx_no_id_used == 0)
    5.20 +  {
    5.21 +    KdPrint((__DRIVER_NAME "     Out of no_id's\n"));    
    5.22      return FREELIST_ID_ERROR;
    5.23 +  }
    5.24    xi->tx_no_id_used++;
    5.25    return 0;
    5.26  }
    5.27 @@ -65,33 +72,17 @@ put_no_id_on_freelist(struct xennet_info
    5.28    xi->tx_no_id_used--;
    5.29  }
    5.30  
    5.31 -static grant_ref_t
    5.32 -get_gref_from_freelist(struct xennet_info *xi)
    5.33 -{
    5.34 -  if (xi->tx_gref_free == 0)
    5.35 -    return 0;
    5.36 -  xi->tx_gref_free--;
    5.37 -  return xi->tx_gref_list[xi->tx_gref_free];
    5.38 -}
    5.39 -
    5.40 -static VOID
    5.41 -put_gref_on_freelist(struct xennet_info *xi, grant_ref_t gref)
    5.42 -{
    5.43 -  ASSERT(xi->tx_page_list[xi->tx_page_free] == NULL);
    5.44 -  xi->tx_gref_list[xi->tx_gref_free] = gref;
    5.45 -  xi->tx_gref_free++;
    5.46 -}
    5.47 -
    5.48  static PMDL
    5.49  get_page_from_freelist(struct xennet_info *xi)
    5.50  {
    5.51    PMDL mdl;
    5.52  
    5.53 -//  KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n"));
    5.54 -
    5.55    if (xi->tx_page_free == 0)
    5.56    {
    5.57 -    mdl = AllocatePage();
    5.58 +    mdl = AllocatePagesExtra(1, sizeof(grant_ref_t));
    5.59 +    *(grant_ref_t *)(((UCHAR *)mdl) + MmSizeOfMdl(0, PAGE_SIZE)) = xi->XenInterface.GntTbl_GrantAccess(
    5.60 +      xi->XenInterface.InterfaceHeader.Context, 0,
    5.61 +      *MmGetMdlPfnArray(mdl), FALSE, 0);
    5.62    }
    5.63    else
    5.64    {
    5.65 @@ -99,121 +90,66 @@ get_page_from_freelist(struct xennet_inf
    5.66      if (xi->tx_page_free < xi->tx_page_free_lowest)
    5.67        xi->tx_page_free_lowest = xi->tx_page_free;
    5.68      mdl = xi->tx_page_list[xi->tx_page_free];
    5.69 -//    KdPrint(("Old Mdl = %p, MmGetMdlVirtualAddress = %p, MmGetSystemAddressForMdlSafe = %p\n",
    5.70 -//      mdl, MmGetMdlVirtualAddress(mdl), MmGetSystemAddressForMdlSafe(mdl, NormalPagePriority)));
    5.71 -    xi->tx_page_list[xi->tx_page_free] = NULL;
    5.72    }
    5.73  
    5.74 -//  KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ "\n"));
    5.75 +  return mdl;
    5.76 +}
    5.77  
    5.78 -  return mdl;
    5.79 +static VOID
    5.80 +put_page_on_freelist(struct xennet_info *xi, PMDL mdl)
    5.81 +{
    5.82 +  xi->tx_page_list[xi->tx_page_free] = mdl;
    5.83 +  xi->tx_page_free++;
    5.84  }
    5.85  
    5.86  static VOID
    5.87  free_page_freelist(struct xennet_info *xi)
    5.88  {
    5.89    PMDL mdl;
    5.90 -//  KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n"));
    5.91 -
    5.92    while(xi->tx_page_free != 0)
    5.93    {
    5.94      xi->tx_page_free--;
    5.95      mdl = xi->tx_page_list[xi->tx_page_free];
    5.96 +    xi->XenInterface.GntTbl_EndAccess(xi->XenInterface.InterfaceHeader.Context,
    5.97 +      *(grant_ref_t *)(((UCHAR *)mdl) + MmSizeOfMdl(0, PAGE_SIZE)), 0);
    5.98      FreePages(mdl);
    5.99    }
   5.100  }
   5.101  
   5.102 -static VOID
   5.103 -put_page_on_freelist(struct xennet_info *xi, PMDL mdl)
   5.104 +static __inline grant_ref_t
   5.105 +get_grant_ref(PMDL mdl)
   5.106  {
   5.107 -//  KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n"));
   5.108 -
   5.109 -//  KdPrint(("Mdl = %p\n",  mdl));
   5.110 -
   5.111 -  xi->tx_page_list[xi->tx_page_free] = mdl;
   5.112 -  xi->tx_page_free++;
   5.113 -
   5.114 -//  KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ "\n"));
   5.115 +  return *(grant_ref_t *)(((UCHAR *)mdl) + MmSizeOfMdl(0, PAGE_SIZE));
   5.116  }
   5.117  
   5.118 -
   5.119  #define SWAP_USHORT(x) (USHORT)((((x & 0xFF) << 8)|((x >> 8) & 0xFF)))
   5.120  
   5.121 -typedef struct
   5.122 -{
   5.123 -  PFN_NUMBER pfn;
   5.124 -  USHORT offset;
   5.125 -  USHORT length;
   5.126 -} page_element_t;
   5.127 -
   5.128 -static VOID
   5.129 -XenNet_BuildPageList(packet_info_t *pi, page_element_t *elements, PUSHORT num_elements)
   5.130 -{
   5.131 -  USHORT element_num = 0;
   5.132 -  UINT offset;
   5.133 -  UINT remaining;
   5.134 -  ULONG pages;
   5.135 -  USHORT page;
   5.136 -  PPFN_NUMBER pfns;
   5.137 -  ULONG i;
   5.138 -
   5.139 -  for (i = 0; i < pi->mdl_count; i++)
   5.140 -  {
   5.141 -    offset = MmGetMdlByteOffset(pi->mdls[i]);
   5.142 -    remaining = MmGetMdlByteCount(pi->mdls[i]);
   5.143 -    pages = ADDRESS_AND_SIZE_TO_SPAN_PAGES(MmGetMdlVirtualAddress(pi->mdls[i]), remaining);
   5.144 -    pfns = MmGetMdlPfnArray(pi->mdls[i]);
   5.145 -    for (page = 0; page < pages; page++, element_num++)
   5.146 -    {
   5.147 -      ASSERT(element_num < *num_elements);
   5.148 -      elements[element_num].pfn = pfns[page];
   5.149 -      elements[element_num].offset = (USHORT)offset;
   5.150 -      elements[element_num].length = (USHORT)min(remaining, PAGE_SIZE - offset);
   5.151 -//KdPrint((__DRIVER_NAME "     adding to page list size = %d, pfn = %08x, offset = %04x\n", elements[element_num].length, elements[element_num].pfn, elements[element_num].offset));
   5.152 -      offset = 0;
   5.153 -      remaining -= elements[element_num].length;
   5.154 -    }
   5.155 -    ASSERT(remaining == 0);
   5.156 -  }
   5.157 -  *num_elements = element_num;
   5.158 -}
   5.159 -
   5.160  /* Place a buffer on tx ring. */
   5.161  static struct netif_tx_request*
   5.162  XenNet_PutOnTxRing(
   5.163    struct xennet_info *xi,
   5.164 -  PFN_NUMBER pfn,
   5.165 -  USHORT offset,
   5.166 -  USHORT len,
   5.167 +  PMDL mdl,
   5.168    uint16_t flags)
   5.169  {
   5.170    struct netif_tx_request *tx;
   5.171 +
   5.172    unsigned short id;
   5.173  
   5.174    id = get_id_from_freelist(xi);
   5.175    ASSERT(id != FREELIST_ID_ERROR);
   5.176    ASSERT(xi->tx_pkts[id] == NULL);
   5.177    tx = RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
   5.178 -
   5.179 -  tx->gref = get_gref_from_freelist(xi);
   5.180 -  ASSERT(tx->gref != 0);
   5.181 -  ASSERT(xi->tx_grefs[id] == 0);
   5.182 -  xi->tx_grefs[id] = tx->gref;
   5.183 -
   5.184 -  xi->XenInterface.GntTbl_GrantAccess(
   5.185 -    xi->XenInterface.InterfaceHeader.Context, 0,
   5.186 -    pfn, FALSE, tx->gref);
   5.187 +  tx->gref = get_grant_ref(mdl);
   5.188 +  xi->tx_mdls[id] = mdl;
   5.189    tx->id = id;
   5.190 -  tx->offset = (uint16_t)offset;
   5.191 -  tx->size = (uint16_t)len;
   5.192 +  tx->offset = 0;
   5.193 +  tx->size = (USHORT)MmGetMdlByteCount(mdl);
   5.194    tx->flags = flags;
   5.195    PC_INC(ProfCount_TxPacketsTotal);
   5.196  
   5.197    return tx;
   5.198  }
   5.199  
   5.200 -static ULONG full_count = 0;
   5.201 -
   5.202  /* Called at DISPATCH_LEVEL with tx_lock held */
   5.203  /*
   5.204   * Send one NDIS_PACKET. This may involve multiple entries on TX ring.
   5.205 @@ -222,19 +158,22 @@ static BOOLEAN
   5.206  XenNet_HWSendPacket(struct xennet_info *xi, PNDIS_PACKET packet)
   5.207  {
   5.208    struct netif_tx_request *tx = NULL;
   5.209 -  struct netif_extra_info *ei;
   5.210 +  struct netif_extra_info *ei = NULL;
   5.211    PNDIS_TCP_IP_CHECKSUM_PACKET_INFO csum_info;
   5.212    UINT total_packet_length;
   5.213 -  ULONG mss; // 0 if not using large send
   5.214 -  PMDL buffer;
   5.215 +  ULONG mss;
   5.216 +  PMDL in_mdl;
   5.217 +  PUCHAR in_buffer = NULL;
   5.218 +  PUCHAR out_buffer;
   5.219 +  USHORT in_remaining;
   5.220 +  USHORT out_remaining;
   5.221    uint16_t flags = NETTXF_more_data;
   5.222 -  page_element_t elements[NET_TX_RING_SIZE];
   5.223 -  USHORT num_elements;
   5.224 -  USHORT element_num;
   5.225    packet_info_t pi;
   5.226 -  PUCHAR address = NULL;
   5.227 -  PMDL merged_buffer = NULL;
   5.228 -  ULONG length = 0;
   5.229 +  BOOLEAN ndis_lso = FALSE;
   5.230 +  BOOLEAN xen_gso = FALSE;
   5.231 +  int pages_required;
   5.232 +  int page_num;
   5.233 +  USHORT copied;
   5.234    
   5.235  #if defined(XEN_PROFILE)
   5.236    LARGE_INTEGER tsc, dummy;
   5.237 @@ -248,61 +187,45 @@ XenNet_HWSendPacket(struct xennet_info *
   5.238      packet, TcpIpChecksumPacketInfo);
   5.239    mss = PtrToUlong(NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo));
   5.240  
   5.241 -  NdisQueryPacket(packet, NULL, NULL, &buffer, &total_packet_length);
   5.242 +  if (mss)
   5.243 +    ndis_lso = TRUE;
   5.244  
   5.245 -  pi.mdls[0] = buffer;
   5.246 -  pi.mdl_count = 1;
   5.247 -  // only if some offload function is in use
   5.248 -  if ((csum_info->Transmit.NdisPacketTcpChecksum
   5.249 -    || csum_info->Transmit.NdisPacketUdpChecksum
   5.250 -    || mss > 0)
   5.251 -    && XenNet_ParsePacketHeader(&pi) == PARSE_TOO_SMALL)
   5.252 +  NdisQueryPacket(packet, NULL, NULL, &in_mdl, &total_packet_length);
   5.253 +
   5.254 +  pages_required = (total_packet_length + PAGE_SIZE - 1) / PAGE_SIZE;
   5.255 +
   5.256 +  if (pages_required + !!ndis_lso > (int)free_requests(xi))
   5.257    {
   5.258 -KdPrint((__DRIVER_NAME "     Split header - merging\n"));
   5.259 -    pi.mdls[0] = merged_buffer = get_page_from_freelist(xi);
   5.260 -    address = MmGetMdlVirtualAddress(pi.mdls[0]);
   5.261 -    memcpy(address, MmGetSystemAddressForMdlSafe(buffer, NormalPagePriority), MmGetMdlByteCount(buffer));
   5.262 -    length = MmGetMdlByteCount(buffer);
   5.263 -    NdisAdjustBufferLength(pi.mdls[0], length); /* do this here so that ParsePacketHeader works */
   5.264 -    while (buffer->Next != NULL && XenNet_ParsePacketHeader(&pi) == PARSE_TOO_SMALL)
   5.265 +    KdPrint((__DRIVER_NAME "     Full on send - required = %d, available = %d\n", pages_required + !!ndis_lso, (int)free_requests(xi)));
   5.266 +    return FALSE;
   5.267 +  }
   5.268 +
   5.269 +  for (page_num = 0, in_remaining = 0; page_num < pages_required; page_num++)
   5.270 +  {
   5.271 +    pi.mdls[page_num] = get_page_from_freelist(xi);
   5.272 +    out_buffer = MmGetMdlVirtualAddress(pi.mdls[page_num]);
   5.273 +    out_remaining = (USHORT)min(PAGE_SIZE, total_packet_length - page_num * PAGE_SIZE);
   5.274 +    NdisAdjustBufferLength(pi.mdls[page_num], out_remaining);
   5.275 +    while (out_remaining > 0)
   5.276      {
   5.277 -      buffer = buffer->Next;
   5.278 -      ASSERT(length + MmGetMdlByteCount(buffer) <= PAGE_SIZE); // I think this could happen
   5.279 -      memcpy(&address[length], MmGetSystemAddressForMdlSafe(buffer, NormalPagePriority), MmGetMdlByteCount(buffer));
   5.280 -      length += MmGetMdlByteCount(buffer);
   5.281 -      NdisAdjustBufferLength(pi.mdls[0], length); /* do this here so that ParsePacketHeader works */
   5.282 -KdPrint((__DRIVER_NAME "     length = %d\n", length));
   5.283 +      if (!in_remaining)
   5.284 +      {
   5.285 +        ASSERT(in_mdl);
   5.286 +        in_buffer = MmGetSystemAddressForMdlSafe(in_mdl, LowPagePriority);
   5.287 +        ASSERT(in_buffer != NULL);
   5.288 +        in_remaining = (USHORT)MmGetMdlByteCount(in_mdl);
   5.289 +      }
   5.290 +      copied = min(in_remaining, out_remaining);
   5.291 +      memcpy(out_buffer, in_buffer, copied);
   5.292 +      in_remaining = in_remaining - copied;
   5.293 +      in_buffer += copied;
   5.294 +      out_remaining = out_remaining - copied;
   5.295 +      out_buffer += copied;
   5.296 +      if (!in_remaining)
   5.297 +        in_mdl = in_mdl->Next;
   5.298      }
   5.299    }
   5.300 -  if (mss > 0 && pi.tcp_length < mss)
   5.301 -  {
   5.302 -    mss = 0;
   5.303 -  }
   5.304 -  ASSERT(buffer != NULL);
   5.305 -  NdisGetNextBuffer(buffer, &buffer);
   5.306 -  while (buffer != NULL)
   5.307 -  {
   5.308 -    ASSERT(pi.mdl_count < MAX_BUFFERS_PER_PACKET);
   5.309 -    pi.mdls[pi.mdl_count++] = buffer;
   5.310 -    NdisGetNextBuffer(buffer, &buffer);
   5.311 -  }
   5.312 -  
   5.313 -  num_elements = NET_TX_RING_SIZE;
   5.314 -  XenNet_BuildPageList(&pi, elements, &num_elements);
   5.315 -
   5.316 -  if (num_elements < pi.mdl_count)
   5.317 -  {
   5.318 -    KdPrint((__DRIVER_NAME "     Less buffers (%d) than mdls (%d)\n", num_elements, pi.mdl_count));
   5.319 -  }
   5.320 -
   5.321 -  if (num_elements + !!mss > (int)free_requests(xi))
   5.322 -  {
   5.323 -//    KdPrint((__DRIVER_NAME "     Full on send - required = %d, available = %d\n", num_elements + !!mss, (int)free_requests(xi)));
   5.324 -    full_count++;
   5.325 -    if (merged_buffer)
   5.326 -      FreePages(merged_buffer);
   5.327 -    return FALSE;
   5.328 -  }
   5.329 +  ASSERT(!in_mdl);
   5.330  
   5.331    if (csum_info->Transmit.NdisPacketTcpChecksum
   5.332      || csum_info->Transmit.NdisPacketUdpChecksum)
   5.333 @@ -311,31 +234,37 @@ KdPrint((__DRIVER_NAME "     length = %d
   5.334      PC_INC(ProfCount_TxPacketsCsumOffload);
   5.335    }
   5.336  
   5.337 -  if (mss > 0)
   5.338 +  if (ndis_lso)
   5.339    {
   5.340 -//    if (!csum_info->Transmit.NdisPacketTcpChecksum)
   5.341 -//      KdPrint((__DRIVER_NAME "     strange... mss set but checksum offload not set\n"));
   5.342 -    flags |= NETTXF_extra_info | NETTXF_csum_blank | NETTXF_data_validated;
   5.343 +    XenNet_ParsePacketHeader(&pi);
   5.344      XenNet_SumIpHeader(MmGetSystemAddressForMdlSafe(pi.mdls[0], NormalPagePriority), pi.ip4_header_length);
   5.345 -    PC_INC(ProfCount_TxPacketsLargeOffload);
   5.346 +    flags |= NETTXF_csum_blank | NETTXF_data_validated; /* these may be implied but not specified when lso is used*/
   5.347 +    if (pi.tcp_length >= mss)
   5.348 +    {
   5.349 +      flags |= NETTXF_extra_info;
   5.350 +      xen_gso = TRUE;
   5.351 +    }
   5.352    }
   5.353  
   5.354    /*
   5.355     * See io/netif.h. Must put (A) 1st request, then (B) optional extra_info, then
   5.356     * (C) rest of requests on the ring. Only (A) has csum flags.
   5.357     */
   5.358 +
   5.359    /* (A) */
   5.360 -  tx = XenNet_PutOnTxRing(xi, elements[0].pfn, elements[0].offset, (USHORT)total_packet_length, flags);
   5.361 +  tx = XenNet_PutOnTxRing(xi, pi.mdls[0], flags);
   5.362 +  tx->size = (USHORT)total_packet_length;
   5.363    xi->tx.req_prod_pvt++;
   5.364  
   5.365    /* (B) */
   5.366 -  if (mss > 0)
   5.367 +  if (xen_gso)
   5.368    {
   5.369 +    ASSERT(flags & NETTXF_extra_info);
   5.370      get_no_id_from_freelist(xi);
   5.371      ei = (struct netif_extra_info *)RING_GET_REQUEST(&xi->tx, xi->tx.req_prod_pvt);
   5.372      ei->type = XEN_NETIF_EXTRA_TYPE_GSO;
   5.373      ei->flags = 0;
   5.374 -    ei->u.gso.size = (USHORT) mss;
   5.375 +    ei->u.gso.size = (USHORT)mss;
   5.376      ei->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
   5.377      ei->u.gso.pad = 0;
   5.378      ei->u.gso.features = 0;
   5.379 @@ -344,37 +273,21 @@ KdPrint((__DRIVER_NAME "     length = %d
   5.380    }
   5.381  
   5.382    /* (C) */
   5.383 -  for (element_num = 1; element_num < num_elements; element_num++)
   5.384 +  for (page_num = 1; page_num < pages_required; page_num++)
   5.385    {
   5.386 -    tx = XenNet_PutOnTxRing(xi, elements[element_num].pfn,
   5.387 -      elements[element_num].offset, elements[element_num].length,
   5.388 -      NETTXF_more_data);
   5.389 +    tx = XenNet_PutOnTxRing(xi, pi.mdls[page_num], NETTXF_more_data);
   5.390      xi->tx.req_prod_pvt++;
   5.391    }
   5.392  
   5.393 -for (element_num = 0, length = 0; element_num < num_elements; element_num++)
   5.394 -{
   5.395 -  if (elements[element_num].length == 0 || elements[element_num].pfn == 0)
   5.396 -    KdPrint((__DRIVER_NAME "     strange page length = %d, pfn = %08x, offset = %04x\n", elements[element_num].length, elements[element_num].pfn, elements[element_num].offset));
   5.397 -  length += elements[element_num].length;
   5.398 -}
   5.399 -if (length != total_packet_length) {
   5.400 -  KdPrint((__DRIVER_NAME "     length (%d) != total_packet_length (%d)\n", length, total_packet_length));
   5.401 -}
   5.402 -
   5.403    /* only set the packet on the last buffer, clear more_data */
   5.404 -  ASSERT(tx);
   5.405    xi->tx_pkts[tx->id] = packet;
   5.406 -  xi->tx_mdls[tx->id] = merged_buffer;
   5.407    tx->flags &= ~NETTXF_more_data;
   5.408  
   5.409 -  if (mss)
   5.410 +  if (ndis_lso)
   5.411    {
   5.412      NDIS_PER_PACKET_INFO_FROM_PACKET(packet, TcpLargeSendPacketInfo) = UlongToPtr(pi.tcp_length);
   5.413    }
   5.414  
   5.415 -  xi->tx_outstanding++;
   5.416 -
   5.417    return TRUE;
   5.418  }
   5.419  
   5.420 @@ -426,8 +339,6 @@ XenNet_SendQueuedPackets(struct xennet_i
   5.421  #endif
   5.422  }
   5.423  
   5.424 -static ULONG ndis_outstanding = 0;
   5.425 -
   5.426  // Called at DISPATCH_LEVEL
   5.427  NDIS_STATUS
   5.428  XenNet_TxBufferGC(struct xennet_info *xi)
   5.429 @@ -457,7 +368,7 @@ XenNet_TxBufferGC(struct xennet_info *xi
   5.430    do {
   5.431      ASSERT(cycles++ < 65536);
   5.432      prod = xi->tx.sring->rsp_prod;
   5.433 -    KeMemoryBarrier(); /* Ensure we see responses up to 'rp'. */
   5.434 +    KeMemoryBarrier(); /* Ensure we see responses up to 'rsp_prod'. */
   5.435  
   5.436      for (cons = xi->tx.rsp_cons; cons != prod; cons++)
   5.437      {
   5.438 @@ -474,21 +385,20 @@ XenNet_TxBufferGC(struct xennet_info *xi
   5.439        }
   5.440  
   5.441        id = txrsp->id;
   5.442 + 
   5.443        packets[packet_count] = xi->tx_pkts[id];
   5.444        if (packets[packet_count])
   5.445        {
   5.446          xi->tx_pkts[id] = NULL;
   5.447          packet_count++;
   5.448          xi->stat_tx_ok++;
   5.449 -        xi->tx_outstanding--;
   5.450        }
   5.451        if (xi->tx_mdls[id])
   5.452        {
   5.453 +        NdisAdjustBufferLength(xi->tx_mdls[id], PAGE_SIZE);
   5.454          put_page_on_freelist(xi, xi->tx_mdls[id]);
   5.455          xi->tx_mdls[id] = NULL;
   5.456        }
   5.457 -      put_gref_on_freelist(xi, xi->tx_grefs[id]);
   5.458 -      xi->tx_grefs[id] = 0;
   5.459        put_id_on_freelist(xi, id);
   5.460      }
   5.461  
   5.462 @@ -507,7 +417,6 @@ XenNet_TxBufferGC(struct xennet_info *xi
   5.463      /* A miniport driver must release any spin lock that it is holding before
   5.464         calling NdisMSendComplete. */
   5.465      NdisMSendComplete(xi->adapter_handle, packets[i], NDIS_STATUS_SUCCESS);
   5.466 -ndis_outstanding--;
   5.467    }
   5.468  
   5.469  //  KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ "\n"));
   5.470 @@ -555,7 +464,6 @@ XenNet_SendPackets(
   5.471  #if defined(XEN_PROFILE)
   5.472      ProfCount_PacketsPerSendPackets++;
   5.473  #endif
   5.474 -ndis_outstanding++;
   5.475    }
   5.476  
   5.477    XenNet_SendQueuedPackets(xi);
   5.478 @@ -608,11 +516,9 @@ XenNet_TxTimer(
   5.479    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
   5.480    KeAcquireSpinLockAtDpcLevel(&xi->tx_lock);
   5.481  
   5.482 -//  KdPrint((__DRIVER_NAME " --- tx_timer - lowest = %d\n", xi->tx_page_free_lowest));
   5.483 +  KdPrint((__DRIVER_NAME " --- tx_timer - tx_page_free_lowest = %d\n", xi->tx_page_free_lowest));
   5.484  
   5.485 -  KdPrint((__DRIVER_NAME " --- tx_outstanding = %d, ndis_outstanding = %d, full_count = %d\n", xi->tx_outstanding, ndis_outstanding, full_count));
   5.486 -  KdPrint((__DRIVER_NAME " --- tx_id_free = %d, xi->tx_no_id_used = %d\n", xi->tx_id_free, xi->tx_no_id_used));
   5.487 -
   5.488 +//  KdPrint((__DRIVER_NAME " --- tx_id_free = %d, xi->tx_no_id_used = %d\n", xi->tx_id_free, xi->tx_no_id_used));
   5.489  
   5.490  #if 0
   5.491    if (xi->tx_page_free_lowest > max(RX_DFL_MIN_TARGET / 4, 16)) // lots of potential for tuning here
   5.492 @@ -634,38 +540,6 @@ XenNet_TxTimer(
   5.493    return;
   5.494  }
   5.495  
   5.496 -static void
   5.497 -XenNet_TxBufferFree(struct xennet_info *xi)
   5.498 -{
   5.499 -  PLIST_ENTRY entry;
   5.500 -  PNDIS_PACKET packet;
   5.501 -  USHORT i;
   5.502 -  grant_ref_t gref;
   5.503 -
   5.504 -  ASSERT(!xi->connected);
   5.505 -
   5.506 -  /* Free packets in tx queue */
   5.507 -  entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
   5.508 -  while (entry != &xi->tx_waiting_pkt_list)
   5.509 -  {
   5.510 -    packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
   5.511 -    NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_FAILURE);
   5.512 -    entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
   5.513 -  }
   5.514 -
   5.515 -  /* free sent-but-not-completed packets */
   5.516 -  for (i = 0; i < NET_TX_RING_SIZE; i++)
   5.517 -  {
   5.518 -    packet = xi->tx_pkts[i];
   5.519 -    if (packet != NULL)
   5.520 -      NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_FAILURE);
   5.521 -    gref = xi->tx_grefs[i];
   5.522 -    if (gref != 0)
   5.523 -      xi->XenInterface.GntTbl_EndAccess(
   5.524 -        xi->XenInterface.InterfaceHeader.Context, gref, TRUE);
   5.525 -  }
   5.526 -}
   5.527 -
   5.528  BOOLEAN
   5.529  XenNet_TxInit(xennet_info_t *xi)
   5.530  {
   5.531 @@ -684,16 +558,8 @@ XenNet_TxInit(xennet_info_t *xi)
   5.532    xi->tx_no_id_used = 0;
   5.533    for (i = 0; i < NET_TX_RING_SIZE; i++)
   5.534    {
   5.535 -    xi->tx_pkts[i] = NULL;
   5.536      put_id_on_freelist(xi, i);
   5.537    }
   5.538 -  xi->tx_gref_free = 0;
   5.539 -  for (i = 0; i < NET_TX_RING_SIZE; i++)
   5.540 -  {
   5.541 -    xi->tx_grefs[i] = 0;
   5.542 -    put_gref_on_freelist(xi, xi->XenInterface.GntTbl_GetRef(
   5.543 -      xi->XenInterface.InterfaceHeader.Context));
   5.544 -  }
   5.545  
   5.546    NdisMInitializeTimer(&xi->tx_timer, xi->adapter_handle, XenNet_TxTimer, xi);
   5.547    NdisMSetPeriodicTimer(&xi->tx_timer, 1000);
   5.548 @@ -701,51 +567,55 @@ XenNet_TxInit(xennet_info_t *xi)
   5.549    return TRUE;
   5.550  }
   5.551  
   5.552 +/*
   5.553 +The ring is completely closed down now. We just need to empty anything left
   5.554 +on our freelists and harvest anything left on the rings.
   5.555 +*/
   5.556 +
   5.557  BOOLEAN
   5.558  XenNet_TxShutdown(xennet_info_t *xi)
   5.559  {
   5.560 -  KIRQL OldIrql;
   5.561 +  PLIST_ENTRY entry;
   5.562 +  PNDIS_PACKET packet;
   5.563 +  PMDL mdl;
   5.564    ULONG i;
   5.565 -  LARGE_INTEGER Interval;
   5.566    BOOLEAN TimerCancelled;
   5.567  
   5.568 -  KeAcquireSpinLock(&xi->tx_lock, &OldIrql);
   5.569 +  KdPrint((__DRIVER_NAME " --> " __FUNCTION__ "\n"));
   5.570  
   5.571    NdisMCancelTimer(&xi->tx_timer, &TimerCancelled);
   5.572  
   5.573 -  XenNet_TxBufferFree(xi);
   5.574 -
   5.575 -  KeReleaseSpinLock(&xi->tx_lock, OldIrql);
   5.576 +  ASSERT(!xi->connected);
   5.577  
   5.578 -  KdPrint((__DRIVER_NAME "     Waiting for tx_outstanding\n"));
   5.579 -  while (xi->tx_outstanding)
   5.580 +  /* Free packets in tx queue */
   5.581 +  entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
   5.582 +  while (entry != &xi->tx_waiting_pkt_list)
   5.583    {
   5.584 -    KdPrint((__DRIVER_NAME "     tx_outstanding = %d\n", xi->tx_outstanding));
   5.585 -    Interval.QuadPart = -10000000;
   5.586 -    KeDelayExecutionThread(KernelMode, FALSE, &Interval);
   5.587 +    packet = CONTAINING_RECORD(entry, NDIS_PACKET, MiniportReservedEx[sizeof(PVOID)]);
   5.588 +    NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_FAILURE);
   5.589 +    entry = RemoveHeadList(&xi->tx_waiting_pkt_list);
   5.590    }
   5.591 -  KdPrint((__DRIVER_NAME "     Done\n"));
   5.592  
   5.593 -  /* free TX resources */
   5.594 -  if (xi->XenInterface.GntTbl_EndAccess(
   5.595 -    xi->XenInterface.InterfaceHeader.Context, xi->tx_ring_ref, 0))
   5.596 +  /* free sent-but-not-completed packets */
   5.597 +  for (i = 0; i < NET_TX_RING_SIZE; i++)
   5.598    {
   5.599 -    xi->tx_ring_ref = GRANT_INVALID_REF;
   5.600 -    FreePages(xi->tx_mdl);
   5.601 +    packet = xi->tx_pkts[i];
   5.602 +    if (packet)
   5.603 +      NdisMSendComplete(xi->adapter_handle, packet, NDIS_STATUS_FAILURE);
   5.604 +    mdl = xi->tx_mdls[i];
   5.605 +    if (mdl)
   5.606 +      put_page_on_freelist(xi, xi->tx_mdls[i]);
   5.607    }
   5.608 -  /* if EndAccess fails then tx/rx ring pages LEAKED -- it's not safe to reuse
   5.609 -     pages Dom0 still has access to */
   5.610 -  xi->tx_pgs = NULL;
   5.611  
   5.612    free_page_freelist(xi);
   5.613  
   5.614 -  ASSERT(xi->tx_outstanding == 0);
   5.615 +  /* free TX resources */
   5.616 +  ASSERT(xi->XenInterface.GntTbl_EndAccess(
   5.617 +    xi->XenInterface.InterfaceHeader.Context, xi->tx_ring_ref, 0));
   5.618 +  FreePages(xi->tx_mdl);
   5.619 +  xi->tx_pgs = NULL;
   5.620  
   5.621 -  for (i = 0; i < NET_TX_RING_SIZE; i++)
   5.622 -  {
   5.623 -    xi->XenInterface.GntTbl_PutRef(
   5.624 -      xi->XenInterface.InterfaceHeader.Context, xi->tx_gref_list[i]);
   5.625 -  }
   5.626 +  KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ "\n"));
   5.627  
   5.628    return TRUE;
   5.629  }
     6.1 --- a/xenpci/evtchn.c	Sat Apr 19 00:36:34 2008 +1000
     6.2 +++ b/xenpci/evtchn.c	Sat Apr 26 16:50:31 2008 +1000
     6.3 @@ -268,7 +268,7 @@ EvtChn_GetXenStoreRingAddr(WDFDEVICE Dev
     6.4    xen_store_mfn = (xen_ulong_t)hvm_get_parameter(Device, HVM_PARAM_STORE_PFN);
     6.5  
     6.6    pa_xen_store_interface.QuadPart = xen_store_mfn << PAGE_SHIFT;
     6.7 -  xen_store_interface = MmMapIoSpace(pa_xen_store_interface, PAGE_SIZE, MmNonCached);
     6.8 +  xen_store_interface = MmMapIoSpace(pa_xen_store_interface, PAGE_SIZE, MmCached);
     6.9  
    6.10    KdPrint((__DRIVER_NAME " xen_store_mfn = %08x\n", xen_store_mfn));
    6.11    //KdPrint((__DRIVER_NAME " xen_store_evtchn = %08x\n", xen_store_evtchn));
     7.1 --- a/xenpci/gnttbl.c	Sat Apr 19 00:36:34 2008 +1000
     7.2 +++ b/xenpci/gnttbl.c	Sat Apr 26 16:50:31 2008 +1000
     7.3 @@ -88,7 +88,7 @@ GntTbl_Init(WDFDEVICE Device)
     7.4    xpdd->gnttab_table_physical = XenPCI_AllocMMIO(Device,
     7.5      PAGE_SIZE * NR_GRANT_FRAMES);
     7.6    xpdd->gnttab_table = MmMapIoSpace(xpdd->gnttab_table_physical,
     7.7 -    PAGE_SIZE * NR_GRANT_FRAMES, MmNonCached);
     7.8 +    PAGE_SIZE * NR_GRANT_FRAMES, MmCached);
     7.9    if (!xpdd->gnttab_table)
    7.10    {
    7.11      KdPrint((__DRIVER_NAME "     Error Mapping Grant Table Shared Memory\n"));
    7.12 @@ -117,6 +117,10 @@ GntTbl_GrantAccess(
    7.13      ref = GntTbl_GetRef(Device);
    7.14    xpdd->gnttab_table[ref].frame = frame;
    7.15    xpdd->gnttab_table[ref].domid = domid;
    7.16 +
    7.17 +  if (xpdd->gnttab_table[ref].flags)
    7.18 +    KdPrint((__DRIVER_NAME "     WARNING: Attempting to re-use grant entry that is already in use!\n"));
    7.19 +
    7.20    KeMemoryBarrier();
    7.21    readonly *= GTF_readonly;
    7.22    xpdd->gnttab_table[ref].flags = GTF_permit_access | (uint16_t)readonly;
     8.1 --- a/xenpci/xenpci.c	Sat Apr 19 00:36:34 2008 +1000
     8.2 +++ b/xenpci/xenpci.c	Sat Apr 26 16:50:31 2008 +1000
     8.3 @@ -888,7 +888,7 @@ XenPci_Suspend(
     8.4      KdPrint((__DRIVER_NAME " <-- " __FUNCTION__ "\n", KeGetCurrentProcessorNumber()));
     8.5      return;
     8.6    }
     8.7 -  ActiveProcessorCount = KeNumberProcessors;
     8.8 +  ActiveProcessorCount = (ULONG)KeNumberProcessors;
     8.9  
    8.10    KdPrint((__DRIVER_NAME "     waiting for all other processors to spin\n"));
    8.11    while (suspend_info->nr_spinning < ActiveProcessorCount - 1)
    8.12 @@ -940,7 +940,7 @@ XenPci_BeginSuspend(WDFDEVICE Device)
    8.13      // TODO: Disable xenbus
    8.14      // TODO: Disable our IRQ
    8.15      //ActiveProcessorCount = KeQueryActiveProcessorCount(&ActiveProcessorMask);
    8.16 -    ActiveProcessorCount = KeNumberProcessors;
    8.17 +    ActiveProcessorCount = (ULONG)KeNumberProcessors;
    8.18      KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
    8.19      for (i = 0; i < ActiveProcessorCount; i++)
    8.20      {