win-pvdrivers

annotate xenpci/xenpci_dma.c @ 685:c13ccf5a629b

Fixed a bug in the dma routines which was causing memory corruption. In some cases when Windows gave an MDL that was longer than the buffer to be dma'd, the end of the buffer would be overwritten. The only time I am aware of this occuring is on one particular map in Call Of Duty 4.

Split out the dma routines from xenpci_pdo.c into xenpci_dma.c
author James Harper <james.harper@bendigoit.com.au>
date Wed Oct 14 14:46:39 2009 +1100 (2009-10-14)
parents
children c3e410ac288f
rev   line source
james@685 1 /*
james@685 2 PV Drivers for Windows Xen HVM Domains
james@685 3 Copyright (C) 2009 James Harper
james@685 4
james@685 5 This program is free software; you can redistribute it and/or
james@685 6 modify it under the terms of the GNU General Public License
james@685 7 as published by the Free Software Foundation; either version 2
james@685 8 of the License, or (at your option) any later version.
james@685 9
james@685 10 This program is distributed in the hope that it will be useful,
james@685 11 but WITHOUT ANY WARRANTY; without even the implied warranty of
james@685 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
james@685 13 GNU General Public License for more details.
james@685 14
james@685 15 You should have received a copy of the GNU General Public License
james@685 16 along with this program; if not, write to the Free Software
james@685 17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
james@685 18 */
james@685 19
james@685 20 #include "xenpci.h"
james@685 21 #include <stdlib.h>
james@685 22 #include <io/ring.h>
james@685 23
james@685 24 #pragma warning(disable : 4200) // zero-sized array
james@685 25 #pragma warning(disable: 4127) // conditional expression is constant
james@685 26
james@685 27 #define MAP_TYPE_VIRTUAL 1
james@685 28 #define MAP_TYPE_MDL 2
james@685 29 #define MAP_TYPE_REMAPPED 3
james@685 30
james@685 31 typedef struct {
james@685 32 ULONG map_type;
james@685 33 PVOID aligned_buffer;
james@685 34 ULONG copy_length;
james@685 35 PMDL mdl;
james@685 36 PVOID currentva;
james@685 37 BOOLEAN allocated_by_me;
james@685 38 } sg_extra_t;
james@685 39
james@685 40 typedef struct {
james@685 41 ULONG map_type;
james@685 42 PVOID aligned_buffer;
james@685 43 PVOID unaligned_buffer;
james@685 44 ULONG copy_length;
james@685 45 PHYSICAL_ADDRESS logical;
james@685 46 } map_register_t;
james@685 47
james@685 48 typedef struct {
james@685 49 PDEVICE_OBJECT device_object;
james@685 50 ULONG total_map_registers;
james@685 51 ULONG count;
james@685 52 map_register_t regs[1];
james@685 53 } map_register_base_t;
james@685 54
james@685 55 BOOLEAN
james@685 56 XenPci_BIS_TranslateBusAddress(PVOID context, PHYSICAL_ADDRESS bus_address, ULONG length, PULONG address_space, PPHYSICAL_ADDRESS translated_address)
james@685 57 {
james@685 58 UNREFERENCED_PARAMETER(context);
james@685 59 UNREFERENCED_PARAMETER(length);
james@685 60 /* actually this isn't right - should look up the gref for the physical address and work backwards from that */
james@685 61 FUNCTION_ENTER();
james@685 62 if (*address_space != 0)
james@685 63 {
james@685 64 KdPrint((__DRIVER_NAME " Cannot map I/O space\n"));
james@685 65 FUNCTION_EXIT();
james@685 66 return FALSE;
james@685 67 }
james@685 68 *translated_address = bus_address;
james@685 69 FUNCTION_EXIT();
james@685 70 return TRUE;
james@685 71 }
james@685 72
james@685 73 static VOID
james@685 74 XenPci_DOP_PutDmaAdapter(PDMA_ADAPTER dma_adapter)
james@685 75 {
james@685 76 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
james@685 77
james@685 78 FUNCTION_ENTER();
james@685 79
james@685 80 if (xen_dma_adapter->dma_extension)
james@685 81 ObDereferenceObject(xen_dma_adapter->dma_extension_driver);
james@685 82 ExFreePoolWithTag(xen_dma_adapter->adapter_object.DmaHeader.DmaOperations, XENPCI_POOL_TAG);
james@685 83 ExFreePoolWithTag(xen_dma_adapter, XENPCI_POOL_TAG);
james@685 84
james@685 85 FUNCTION_EXIT();
james@685 86
james@685 87 return;
james@685 88 }
james@685 89
james@685 90 static PVOID
james@685 91 XenPci_DOP_AllocateCommonBuffer(
james@685 92 PDMA_ADAPTER DmaAdapter,
james@685 93 ULONG Length,
james@685 94 PPHYSICAL_ADDRESS LogicalAddress,
james@685 95 BOOLEAN CacheEnabled
james@685 96 )
james@685 97 {
james@685 98 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)DmaAdapter;
james@685 99 PXENPCI_DEVICE_DATA xpdd;
james@685 100 PVOID buffer;
james@685 101 PFN_NUMBER pfn;
james@685 102 grant_ref_t gref;
james@685 103
james@685 104 UNREFERENCED_PARAMETER(DmaAdapter);
james@685 105 UNREFERENCED_PARAMETER(CacheEnabled);
james@685 106
james@685 107 //FUNCTION_ENTER();
james@685 108
james@685 109 xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
james@685 110
james@685 111 //KdPrint((__DRIVER_NAME " Length = %d\n", Length));
james@685 112
james@685 113 buffer = ExAllocatePoolWithTag(NonPagedPool, Length, XENPCI_POOL_TAG);
james@685 114 ASSERT(buffer); /* lazy */
james@685 115
james@685 116 pfn = (PFN_NUMBER)(MmGetPhysicalAddress(buffer).QuadPart >> PAGE_SHIFT);
james@685 117 ASSERT(pfn); /* lazy */
james@685 118 gref = (grant_ref_t)GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE, INVALID_GRANT_REF);
james@685 119 ASSERT(gref != INVALID_GRANT_REF); /* lazy */
james@685 120 LogicalAddress->QuadPart = (gref << PAGE_SHIFT) | (PtrToUlong(buffer) & (PAGE_SIZE - 1));
james@685 121
james@685 122 //FUNCTION_EXIT();
james@685 123 return buffer;
james@685 124 }
james@685 125
james@685 126 static VOID
james@685 127 XenPci_DOP_FreeCommonBuffer(
james@685 128 PDMA_ADAPTER dma_adapter,
james@685 129 ULONG length,
james@685 130 PHYSICAL_ADDRESS logical_address,
james@685 131 PVOID virtual_address,
james@685 132 BOOLEAN cache_enabled
james@685 133 )
james@685 134 {
james@685 135 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
james@685 136 PXENPCI_DEVICE_DATA xpdd;
james@685 137 grant_ref_t gref;
james@685 138
james@685 139 UNREFERENCED_PARAMETER(dma_adapter);
james@685 140 UNREFERENCED_PARAMETER(length);
james@685 141 UNREFERENCED_PARAMETER(cache_enabled);
james@685 142
james@685 143 // FUNCTION_ENTER();
james@685 144
james@685 145 xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
james@685 146 gref = (grant_ref_t)(logical_address.QuadPart >> PAGE_SHIFT);
james@685 147 //KdPrint((__DRIVER_NAME " F Releasing Grant Ref %d\n", gref));
james@685 148 GntTbl_EndAccess(xpdd, gref, FALSE);
james@685 149 //KdPrint((__DRIVER_NAME " F Released Grant Ref\n"));
james@685 150 ExFreePoolWithTag(virtual_address, XENPCI_POOL_TAG);
james@685 151
james@685 152 // FUNCTION_EXIT();
james@685 153 }
james@685 154
james@685 155 static NTSTATUS
james@685 156 XenPci_DOP_AllocateAdapterChannel(
james@685 157 IN PDMA_ADAPTER dma_adapter,
james@685 158 IN PDEVICE_OBJECT device_object,
james@685 159 IN ULONG NumberOfMapRegisters,
james@685 160 IN PDRIVER_CONTROL ExecutionRoutine,
james@685 161 IN PVOID Context
james@685 162 )
james@685 163 {
james@685 164 //xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
james@685 165 //PXENPCI_DEVICE_DATA xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
james@685 166 IO_ALLOCATION_ACTION action;
james@685 167 map_register_base_t *map_register_base;
james@685 168
james@685 169 UNREFERENCED_PARAMETER(dma_adapter);
james@685 170
james@685 171 //FUNCTION_ENTER();
james@685 172
james@685 173 map_register_base = ExAllocatePoolWithTag(NonPagedPool,
james@685 174 FIELD_OFFSET(map_register_base_t, regs) + NumberOfMapRegisters * sizeof(map_register_t), XENPCI_POOL_TAG);
james@685 175 if (!map_register_base)
james@685 176 {
james@685 177 KdPrint((__DRIVER_NAME " Cannot allocate memory for map_register_base\n"));
james@685 178 //FUNCTION_EXIT();
james@685 179 return STATUS_INSUFFICIENT_RESOURCES;
james@685 180 }
james@685 181 /* we should also allocate a single page of memory here for remap purposes as once we allocate the map registers there is no failure allowed */
james@685 182 map_register_base->device_object = device_object;
james@685 183 map_register_base->total_map_registers = NumberOfMapRegisters;
james@685 184 map_register_base->count = 0;
james@685 185
james@685 186 action = ExecutionRoutine(device_object, device_object->CurrentIrp, map_register_base, Context);
james@685 187
james@685 188 switch (action)
james@685 189 {
james@685 190 case KeepObject:
james@685 191 KdPrint((__DRIVER_NAME " KeepObject\n"));
james@685 192 ASSERT(FALSE);
james@685 193 break;
james@685 194 case DeallocateObject:
james@685 195 KdPrint((__DRIVER_NAME " DeallocateObject\n"));
james@685 196 ASSERT(FALSE);
james@685 197 break;
james@685 198 case DeallocateObjectKeepRegisters:
james@685 199 //KdPrint((__DRIVER_NAME " DeallocateObjectKeepRegisters\n"));
james@685 200 break;
james@685 201 default:
james@685 202 KdPrint((__DRIVER_NAME " Unknown action %d\n", action));
james@685 203 ASSERT(FALSE);
james@685 204 break;
james@685 205 }
james@685 206 //FUNCTION_EXIT();
james@685 207 return STATUS_SUCCESS;
james@685 208 }
james@685 209
james@685 210 static BOOLEAN
james@685 211 XenPci_DOP_FlushAdapterBuffers(
james@685 212 PDMA_ADAPTER dma_adapter,
james@685 213 PMDL mdl,
james@685 214 PVOID MapRegisterBase,
james@685 215 PVOID CurrentVa,
james@685 216 ULONG Length,
james@685 217 BOOLEAN write_to_device)
james@685 218 {
james@685 219 //xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
james@685 220 //PXENPCI_DEVICE_DATA xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
james@685 221 map_register_base_t *map_register_base = MapRegisterBase;
james@685 222 map_register_t *map_register;
james@685 223 ULONG i;
james@685 224
james@685 225 UNREFERENCED_PARAMETER(dma_adapter);
james@685 226 UNREFERENCED_PARAMETER(mdl);
james@685 227 UNREFERENCED_PARAMETER(CurrentVa);
james@685 228 UNREFERENCED_PARAMETER(Length);
james@685 229
james@685 230 //FUNCTION_ENTER();
james@685 231
james@685 232 for (i = 0; i < map_register_base->count; i++)
james@685 233 {
james@685 234 map_register = &map_register_base->regs[i];
james@685 235 if (map_register->map_type == MAP_TYPE_REMAPPED && !write_to_device)
james@685 236 memcpy(map_register->unaligned_buffer, map_register->aligned_buffer, map_register->copy_length);
james@685 237 }
james@685 238 //FUNCTION_EXIT();
james@685 239
james@685 240 return TRUE;
james@685 241 }
james@685 242
james@685 243 static VOID
james@685 244 XenPci_DOP_FreeAdapterChannel(
james@685 245 IN PDMA_ADAPTER DmaAdapter
james@685 246 )
james@685 247 {
james@685 248 UNREFERENCED_PARAMETER(DmaAdapter);
james@685 249
james@685 250 FUNCTION_ENTER();
james@685 251 FUNCTION_EXIT();
james@685 252 }
james@685 253
james@685 254 static VOID
james@685 255 XenPci_DOP_FreeMapRegisters(
james@685 256 PDMA_ADAPTER dma_adapter,
james@685 257 PVOID MapRegisterBase,
james@685 258 ULONG NumberOfMapRegisters)
james@685 259 {
james@685 260 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
james@685 261 PXENPCI_DEVICE_DATA xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
james@685 262 map_register_base_t *map_register_base = MapRegisterBase;
james@685 263 map_register_t *map_register;
james@685 264 ULONG i;
james@685 265 grant_ref_t gref;
james@685 266
james@685 267 //FUNCTION_ENTER();
james@685 268 if (!map_register_base)
james@685 269 {
james@685 270 /* i'm not sure if this is ideal here, but NDIS definitely does it */
james@685 271 return;
james@685 272 }
james@685 273 ASSERT(map_register_base->total_map_registers == NumberOfMapRegisters);
james@685 274
james@685 275 for (i = 0; i < map_register_base->count; i++)
james@685 276 {
james@685 277 map_register = &map_register_base->regs[i];
james@685 278 switch (map_register->map_type)
james@685 279 {
james@685 280 case MAP_TYPE_REMAPPED:
james@685 281 gref = (grant_ref_t)(map_register->logical.QuadPart >> PAGE_SHIFT);
james@685 282 GntTbl_EndAccess(xpdd, gref, FALSE);
james@685 283 ExFreePoolWithTag(map_register->aligned_buffer, XENPCI_POOL_TAG);
james@685 284 break;
james@685 285 case MAP_TYPE_MDL:
james@685 286 gref = (grant_ref_t)(map_register->logical.QuadPart >> PAGE_SHIFT);
james@685 287 GntTbl_EndAccess(xpdd, gref, FALSE);
james@685 288 break;
james@685 289 case MAP_TYPE_VIRTUAL:
james@685 290 break;
james@685 291 }
james@685 292 }
james@685 293 ExFreePoolWithTag(map_register_base, XENPCI_POOL_TAG);
james@685 294
james@685 295 //FUNCTION_EXIT();
james@685 296 }
james@685 297
james@685 298 static PHYSICAL_ADDRESS
james@685 299 XenPci_DOP_MapTransfer(
james@685 300 PDMA_ADAPTER dma_adapter,
james@685 301 PMDL mdl,
james@685 302 PVOID MapRegisterBase,
james@685 303 PVOID CurrentVa,
james@685 304 PULONG Length,
james@685 305 BOOLEAN WriteToDevice)
james@685 306 {
james@685 307 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
james@685 308 PXENPCI_DEVICE_DATA xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
james@685 309 map_register_base_t *map_register_base = MapRegisterBase;
james@685 310 map_register_t *map_register = &map_register_base->regs[map_register_base->count];
james@685 311 PDEVICE_OBJECT device_object = map_register_base->device_object;
james@685 312 ULONG page_offset;
james@685 313 PFN_NUMBER pfn;
james@685 314 grant_ref_t gref;
james@685 315 PUCHAR ptr;
james@685 316 ULONG mdl_offset;
james@685 317 ULONG pfn_index;
james@685 318
james@685 319 //FUNCTION_ENTER();
james@685 320
james@685 321 //KdPrint((__DRIVER_NAME " Mdl = %p, MapRegisterBase = %p, MdlVa = %p, CurrentVa = %p, Length = %d\n",
james@685 322 // mdl, MapRegisterBase, MmGetMdlVirtualAddress(mdl), CurrentVa, *Length));
james@685 323
james@685 324 ASSERT(mdl);
james@685 325 ASSERT(map_register_base);
james@685 326 ASSERT(map_register_base->count < map_register_base->total_map_registers);
james@685 327
james@685 328 if (xen_dma_adapter->dma_extension)
james@685 329 {
james@685 330 if (xen_dma_adapter->dma_extension->need_virtual_address && xen_dma_adapter->dma_extension->need_virtual_address(device_object->CurrentIrp))
james@685 331 {
james@685 332 map_register->map_type = MAP_TYPE_VIRTUAL;
james@685 333 }
james@685 334 else
james@685 335 {
james@685 336 if (xen_dma_adapter->dma_extension->get_alignment)
james@685 337 {
james@685 338 ULONG alignment = xen_dma_adapter->dma_extension->get_alignment(device_object->CurrentIrp);
james@685 339 if ((MmGetMdlByteOffset(mdl) & (alignment - 1)) || (MmGetMdlByteCount(mdl) & (alignment - 1)))
james@685 340 {
james@685 341 map_register->map_type = MAP_TYPE_REMAPPED;
james@685 342 }
james@685 343 else
james@685 344 {
james@685 345 map_register->map_type = MAP_TYPE_MDL;
james@685 346 }
james@685 347 }
james@685 348 else
james@685 349 {
james@685 350 map_register->map_type = MAP_TYPE_MDL;
james@685 351 }
james@685 352 }
james@685 353 }
james@685 354 else
james@685 355 {
james@685 356 map_register->map_type = MAP_TYPE_MDL;
james@685 357 }
james@685 358
james@685 359 switch (map_register->map_type)
james@685 360 {
james@685 361 case MAP_TYPE_MDL:
james@685 362 //KdPrint((__DRIVER_NAME " MAP_TYPE_MDL\n"));
james@685 363 mdl_offset = (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)MmGetMdlVirtualAddress(mdl));
james@685 364 page_offset = PtrToUlong(CurrentVa) & (PAGE_SIZE - 1);
james@685 365 *Length = min(*Length, PAGE_SIZE - page_offset);
james@685 366 pfn_index = (ULONG)(((UINT_PTR)CurrentVa >> PAGE_SHIFT) - ((UINT_PTR)MmGetMdlVirtualAddress(mdl) >> PAGE_SHIFT));
james@685 367 //KdPrint((__DRIVER_NAME " mdl_offset = %d, page_offset = %d, length = %d, pfn_index = %d\n",
james@685 368 // mdl_offset, page_offset, *Length, pfn_index));
james@685 369 pfn = MmGetMdlPfnArray(mdl)[pfn_index];
james@685 370 //KdPrint((__DRIVER_NAME " B Requesting Grant Ref\n"));
james@685 371 gref = (grant_ref_t)GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE, INVALID_GRANT_REF);
james@685 372 //KdPrint((__DRIVER_NAME " B Got Grant Ref %d\n", gref));
james@685 373 map_register->logical.QuadPart = (LONGLONG)(gref << PAGE_SHIFT) | page_offset;
james@685 374 map_register_base->count++;
james@685 375 break;
james@685 376 case MAP_TYPE_REMAPPED:
james@685 377 //KdPrint((__DRIVER_NAME " MAP_TYPE_REMAPPED (MapTransfer)\n"));
james@685 378 //KdPrint((__DRIVER_NAME " Mdl = %p, MapRegisterBase = %p, MdlVa = %p, CurrentVa = %p, Length = %d\n",
james@685 379 // mdl, MapRegisterBase, MmGetMdlVirtualAddress(mdl), CurrentVa, *Length));
james@685 380 mdl_offset = (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)MmGetMdlVirtualAddress(mdl));
james@685 381 *Length = min(*Length, PAGE_SIZE);
james@685 382 map_register->aligned_buffer = ExAllocatePoolWithTag(NonPagedPool, PAGE_SIZE, XENPCI_POOL_TAG);
james@685 383 ASSERT(map_register->aligned_buffer);
james@685 384 map_register->unaligned_buffer = MmGetSystemAddressForMdlSafe(mdl, NormalPagePriority);
james@685 385 ASSERT(map_register->unaligned_buffer); /* lazy */
james@685 386 map_register->unaligned_buffer = (PUCHAR)map_register->unaligned_buffer + mdl_offset;
james@685 387 map_register->copy_length = *Length;
james@685 388 if (WriteToDevice)
james@685 389 memcpy(map_register->aligned_buffer, map_register->unaligned_buffer, map_register->copy_length);
james@685 390 pfn = (PFN_NUMBER)(MmGetPhysicalAddress(map_register->aligned_buffer).QuadPart >> PAGE_SHIFT);
james@685 391 //KdPrint((__DRIVER_NAME " C Requesting Grant Ref\n"));
james@685 392 gref = (grant_ref_t)GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE, INVALID_GRANT_REF);
james@685 393 //KdPrint((__DRIVER_NAME " C Got Grant Ref %d\n", gref));
james@685 394 map_register->logical.QuadPart = (LONGLONG)(gref << PAGE_SHIFT);
james@685 395 map_register_base->count++;
james@685 396 break;
james@685 397 case MAP_TYPE_VIRTUAL:
james@685 398 //KdPrint((__DRIVER_NAME " MAP_TYPE_VIRTUAL\n"));
james@685 399 ptr = MmGetSystemAddressForMdlSafe(mdl, NormalPagePriority);
james@685 400 ASSERT(ptr); /* lazy */
james@685 401 map_register->logical.QuadPart = (ULONGLONG)ptr;
james@685 402 map_register_base->count++;
james@685 403 break;
james@685 404 default:
james@685 405 ASSERT(FALSE);
james@685 406 break;
james@685 407 }
james@685 408
james@685 409 //KdPrint((__DRIVER_NAME " logical = %08x:%08x\n", map_register->logical.HighPart, map_register->logical.LowPart));
james@685 410 //FUNCTION_EXIT();
james@685 411 return map_register->logical;
james@685 412 }
james@685 413
james@685 414 static ULONG
james@685 415 XenPci_DOP_GetDmaAlignment(
james@685 416 PDMA_ADAPTER DmaAdapter)
james@685 417 {
james@685 418 UNREFERENCED_PARAMETER(DmaAdapter);
james@685 419
james@685 420 FUNCTION_ENTER();
james@685 421 FUNCTION_EXIT();
james@685 422 return 0;
james@685 423 }
james@685 424
james@685 425 static ULONG
james@685 426 XenPci_DOP_ReadDmaCounter(
james@685 427 PDMA_ADAPTER DmaAdapter)
james@685 428 {
james@685 429 UNREFERENCED_PARAMETER(DmaAdapter);
james@685 430
james@685 431 FUNCTION_ENTER();
james@685 432 FUNCTION_EXIT();
james@685 433 return 0;
james@685 434 }
james@685 435
james@685 436 static VOID
james@685 437 XenPci_DOP_PutScatterGatherList(
james@685 438 IN PDMA_ADAPTER DmaAdapter,
james@685 439 IN PSCATTER_GATHER_LIST sg_list,
james@685 440 IN BOOLEAN WriteToDevice
james@685 441 )
james@685 442 {
james@685 443 xen_dma_adapter_t *xen_dma_adapter;
james@685 444 PXENPCI_DEVICE_DATA xpdd;
james@685 445 ULONG i;
james@685 446 sg_extra_t *sg_extra;
james@685 447 PMDL curr_mdl;
james@685 448 ULONG offset;
james@685 449 BOOLEAN active;
james@685 450
james@685 451 UNREFERENCED_PARAMETER(WriteToDevice);
james@685 452
james@685 453 //FUNCTION_ENTER();
james@685 454
james@685 455 xen_dma_adapter = (xen_dma_adapter_t *)DmaAdapter;
james@685 456 ASSERT(xen_dma_adapter);
james@685 457 xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
james@685 458
james@685 459 sg_extra = (sg_extra_t *)((PUCHAR)sg_list + FIELD_OFFSET(SCATTER_GATHER_LIST, Elements) +
james@685 460 (sizeof(SCATTER_GATHER_ELEMENT)) * sg_list->NumberOfElements);
james@685 461
james@685 462 switch (sg_extra->map_type)
james@685 463 {
james@685 464 case MAP_TYPE_REMAPPED:
james@685 465 for (i = 0; i < sg_list->NumberOfElements; i++)
james@685 466 {
james@685 467 grant_ref_t gref;
james@685 468 gref = (grant_ref_t)(sg_list->Elements[i].Address.QuadPart >> PAGE_SHIFT);
james@685 469 GntTbl_EndAccess(xpdd, gref, FALSE);
james@685 470 sg_list->Elements[i].Address.QuadPart = -1;
james@685 471 }
james@685 472 ASSERT(sg_extra->mdl);
james@685 473 if (!WriteToDevice)
james@685 474 {
james@685 475 for (curr_mdl = sg_extra->mdl, offset = 0, active = FALSE; curr_mdl && offset < sg_extra->copy_length; curr_mdl = curr_mdl->Next)
james@685 476 {
james@685 477 PVOID mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
james@685 478 ULONG mdl_byte_count = MmGetMdlByteCount(curr_mdl);
james@685 479 ULONG mdl_offset = 0;
james@685 480 /* need to use <= va + len - 1 to avoid ptr wraparound */
james@685 481 if ((UINT_PTR)sg_extra->currentva >= (UINT_PTR)mdl_start_va && (UINT_PTR)sg_extra->currentva <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
james@685 482 {
james@685 483 active = TRUE;
james@685 484 mdl_byte_count -= (ULONG)((UINT_PTR)sg_extra->currentva - (UINT_PTR)mdl_start_va);
james@685 485 if (offset + mdl_byte_count > sg_extra->copy_length)
james@685 486 mdl_byte_count = sg_extra->copy_length - offset;
james@685 487 mdl_offset = (ULONG)((UINT_PTR)sg_extra->currentva - (UINT_PTR)mdl_start_va);
james@685 488 mdl_start_va = sg_extra->currentva;
james@685 489 }
james@685 490 if (active)
james@685 491 {
james@685 492 PVOID unaligned_buffer;
james@685 493 unaligned_buffer = MmGetSystemAddressForMdlSafe(curr_mdl, NormalPagePriority);
james@685 494 ASSERT(unaligned_buffer); /* lazy */
james@685 495 memcpy((PUCHAR)unaligned_buffer + mdl_offset, (PUCHAR)sg_extra->aligned_buffer + offset, mdl_byte_count);
james@685 496 offset += mdl_byte_count;
james@685 497 }
james@685 498 }
james@685 499 ASSERT(offset == sg_extra->copy_length);
james@685 500 }
james@685 501 ExFreePoolWithTag(sg_extra->aligned_buffer, XENPCI_POOL_TAG);
james@685 502 break;
james@685 503 case MAP_TYPE_MDL:
james@685 504 for (i = 0; i < sg_list->NumberOfElements; i++)
james@685 505 {
james@685 506 grant_ref_t gref;
james@685 507 gref = (grant_ref_t)(sg_list->Elements[i].Address.QuadPart >> PAGE_SHIFT);
james@685 508 GntTbl_EndAccess(xpdd, gref, FALSE);
james@685 509 sg_list->Elements[i].Address.QuadPart = -1;
james@685 510 }
james@685 511 break;
james@685 512 case MAP_TYPE_VIRTUAL:
james@685 513 break;
james@685 514 }
james@685 515 if (sg_extra->allocated_by_me)
james@685 516 ExFreePoolWithTag(sg_list, XENPCI_POOL_TAG);
james@685 517 //FUNCTION_EXIT();
james@685 518 }
james@685 519
james@685 520 static NTSTATUS
james@685 521 XenPci_DOP_CalculateScatterGatherList(
james@685 522 PDMA_ADAPTER DmaAdapter,
james@685 523 PMDL Mdl,
james@685 524 PVOID CurrentVa,
james@685 525 ULONG Length,
james@685 526 PULONG ScatterGatherListSize,
james@685 527 PULONG NumberOfMapRegisters
james@685 528 )
james@685 529 {
james@685 530 xen_dma_adapter_t *xen_dma_adapter;
james@685 531 ULONG elements;
james@685 532 PMDL curr_mdl;
james@685 533
james@685 534 UNREFERENCED_PARAMETER(CurrentVa);
james@685 535
james@685 536 //FUNCTION_ENTER();
james@685 537
james@685 538 //KdPrint((__DRIVER_NAME " Mdl = %p\n", Mdl));
james@685 539 //KdPrint((__DRIVER_NAME " CurrentVa = %p\n", CurrentVa));
james@685 540 //KdPrint((__DRIVER_NAME " Length = %d\n", Length));
james@685 541
james@685 542 xen_dma_adapter = (xen_dma_adapter_t *)DmaAdapter;
james@685 543
james@685 544 if (Mdl)
james@685 545 {
james@685 546 //if (CurrentVa != MmGetMdlVirtualAddress(Mdl))
james@685 547 //{
james@685 548 // KdPrint((__DRIVER_NAME " CurrentVa (%p) != MdlVa (%p)\n", CurrentVa, MmGetMdlVirtualAddress(Mdl)));
james@685 549 //
james@685 550
james@685 551 //KdPrint((__DRIVER_NAME " CurrentVa = %p, MdlVa = %p\n", CurrentVa, MmGetMdlVirtualAddress(Mdl)));
james@685 552
james@685 553 for (curr_mdl = Mdl, elements = 0; curr_mdl; curr_mdl = curr_mdl->Next)
james@685 554 {
james@685 555 //KdPrint((__DRIVER_NAME " curr_mdlVa = %p, curr_mdl size = %d\n", MmGetMdlVirtualAddress(curr_mdl), MmGetMdlByteCount(curr_mdl)));
james@685 556 elements += ADDRESS_AND_SIZE_TO_SPAN_PAGES(MmGetMdlVirtualAddress(curr_mdl), MmGetMdlByteCount(curr_mdl));
james@685 557 }
james@685 558 }
james@685 559 else
james@685 560 {
james@685 561 elements = ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, Length); // + 1;
james@685 562 }
james@685 563
james@685 564 if (elements > xen_dma_adapter->adapter_object.MapRegistersPerChannel)
james@685 565 {
james@685 566 //KdPrint((__DRIVER_NAME " elements = %d - too many\n", elements));
james@685 567 if (NumberOfMapRegisters)
james@685 568 *NumberOfMapRegisters = 0;
james@685 569 *ScatterGatherListSize = 0;
james@685 570
james@685 571 return STATUS_INSUFFICIENT_RESOURCES;
james@685 572 }
james@685 573
james@685 574 *ScatterGatherListSize = FIELD_OFFSET(SCATTER_GATHER_LIST, Elements)
james@685 575 + sizeof(SCATTER_GATHER_ELEMENT) * elements
james@685 576 + sizeof(sg_extra_t);
james@685 577 if (NumberOfMapRegisters)
james@685 578 *NumberOfMapRegisters = elements;
james@685 579
james@685 580 //KdPrint((__DRIVER_NAME " ScatterGatherListSize = %d, NumberOfMapRegisters = %d\n", *ScatterGatherListSize, elements));
james@685 581
james@685 582 //FUNCTION_EXIT();
james@685 583 return STATUS_SUCCESS;
james@685 584 }
james@685 585
james@685 586 static NTSTATUS
james@685 587 XenPci_DOP_BuildScatterGatherListButDontExecute(
james@685 588 IN PDMA_ADAPTER DmaAdapter,
james@685 589 IN PDEVICE_OBJECT DeviceObject,
james@685 590 IN PMDL Mdl,
james@685 591 IN PVOID CurrentVa,
james@685 592 IN ULONG Length,
james@685 593 IN BOOLEAN WriteToDevice,
james@685 594 IN PVOID ScatterGatherBuffer,
james@685 595 IN ULONG ScatterGatherBufferLength,
james@685 596 BOOLEAN allocated_by_me)
james@685 597 {
james@685 598 ULONG i;
james@685 599 PSCATTER_GATHER_LIST sglist = ScatterGatherBuffer;
james@685 600 PUCHAR ptr;
james@685 601 ULONG remaining = Length;
james@685 602 ULONG total_remaining;
james@685 603 xen_dma_adapter_t *xen_dma_adapter;
james@685 604 PXENPCI_DEVICE_DATA xpdd;
james@685 605 sg_extra_t *sg_extra;
james@685 606 PMDL curr_mdl;
james@685 607 ULONG map_type;
james@685 608 ULONG sg_element;
james@685 609 ULONG offset;
james@685 610 PFN_NUMBER pfn;
james@685 611 grant_ref_t gref;
james@685 612 BOOLEAN active;
james@685 613 PVOID mdl_start_va;
james@685 614 ULONG mdl_byte_count;
james@685 615 ULONG mdl_offset;
james@685 616 ULONG remapped_bytes = 0;
james@685 617
james@685 618 //FUNCTION_ENTER();
james@685 619
james@685 620 if (!ScatterGatherBuffer)
james@685 621 {
james@685 622 KdPrint((__DRIVER_NAME " NULL ScatterGatherBuffer\n"));
james@685 623 return STATUS_INVALID_PARAMETER;
james@685 624 }
james@685 625 //if (MmGetMdlVirtualAddress(Mdl) != CurrentVa)
james@685 626 //{
james@685 627 // KdPrint((__DRIVER_NAME " MmGetMdlVirtualAddress = %p, CurrentVa = %p, Length = %d\n", MmGetMdlVirtualAddress(Mdl), CurrentVa, Length));
james@685 628 //}
james@685 629
james@685 630 xen_dma_adapter = (xen_dma_adapter_t *)DmaAdapter;
james@685 631 xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
james@685 632
james@685 633 ASSERT(Mdl);
james@685 634
james@685 635 if (xen_dma_adapter->dma_extension)
james@685 636 {
james@685 637 if (xen_dma_adapter->dma_extension->need_virtual_address && xen_dma_adapter->dma_extension->need_virtual_address(DeviceObject->CurrentIrp))
james@685 638 {
james@685 639 ASSERT(!Mdl->Next); /* can only virtual a single buffer */
james@685 640 //ASSERT(MmGetMdlVirtualAddress(Mdl) == CurrentVa);
james@685 641 map_type = MAP_TYPE_VIRTUAL;
james@685 642 sglist->NumberOfElements = 1;
james@685 643 }
james@685 644 else
james@685 645 {
james@685 646 if (xen_dma_adapter->dma_extension->get_alignment)
james@685 647 {
james@685 648 ULONG alignment = xen_dma_adapter->dma_extension->get_alignment(DeviceObject->CurrentIrp);
james@685 649
james@685 650 map_type = MAP_TYPE_MDL;
james@685 651 sglist->NumberOfElements = 0;
james@685 652 for (curr_mdl = Mdl, remapped_bytes = 0, active = FALSE; remapped_bytes < Length && curr_mdl; curr_mdl = curr_mdl->Next)
james@685 653 {
james@685 654 mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
james@685 655 mdl_byte_count = MmGetMdlByteCount(curr_mdl);
james@685 656 /* need to use <= va + len - 1 to avoid ptr wraparound */
james@685 657 if ((UINT_PTR)CurrentVa >= (UINT_PTR)mdl_start_va && (UINT_PTR)CurrentVa <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
james@685 658 {
james@685 659 active = TRUE;
james@685 660 mdl_byte_count -= (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
james@685 661 if (remapped_bytes + mdl_byte_count > Length)
james@685 662 mdl_byte_count = Length - remapped_bytes;
james@685 663 mdl_start_va = CurrentVa;
james@685 664 }
james@685 665 if (active)
james@685 666 {
james@685 667 if (((UINT_PTR)mdl_start_va & (alignment - 1)) || (mdl_byte_count & (alignment - 1)))
james@685 668 map_type = MAP_TYPE_REMAPPED;
james@685 669 remapped_bytes += mdl_byte_count;
james@685 670 if (remapped_bytes > Length)
james@685 671 remapped_bytes = Length;
james@685 672 }
james@685 673 }
james@685 674 if (remapped_bytes != Length)
james@685 675 {
james@685 676 KdPrint((__DRIVER_NAME " remapped_bytes = %d, Length = %d\n", remapped_bytes, Length));
james@685 677 }
james@685 678 //ASSERT(remapped_bytes == Length);
james@685 679 sglist->NumberOfElements += ADDRESS_AND_SIZE_TO_SPAN_PAGES(NULL, remapped_bytes);
james@685 680 }
james@685 681 else
james@685 682 {
james@685 683 map_type = MAP_TYPE_MDL;
james@685 684 }
james@685 685 }
james@685 686 }
james@685 687 else
james@685 688 {
james@685 689 map_type = MAP_TYPE_MDL;
james@685 690 }
james@685 691 if (map_type == MAP_TYPE_MDL)
james@685 692 {
james@685 693 for (curr_mdl = Mdl, sglist->NumberOfElements = 0, total_remaining = Length, active = FALSE; total_remaining > 0; curr_mdl = curr_mdl->Next)
james@685 694 {
james@685 695 ASSERT(curr_mdl);
james@685 696 mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
james@685 697 mdl_byte_count = MmGetMdlByteCount(curr_mdl);
james@685 698 /* need to use <= va + len - 1 to avoid ptr wraparound */
james@685 699 if ((UINT_PTR)CurrentVa >= (UINT_PTR)mdl_start_va && (UINT_PTR)CurrentVa <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
james@685 700 {
james@685 701 active = TRUE;
james@685 702 mdl_byte_count -= (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
james@685 703 mdl_start_va = CurrentVa;
james@685 704 }
james@685 705 mdl_byte_count = min(mdl_byte_count, total_remaining);
james@685 706 if (active && mdl_byte_count)
james@685 707 {
james@685 708 sglist->NumberOfElements += ADDRESS_AND_SIZE_TO_SPAN_PAGES(
james@685 709 mdl_start_va, mdl_byte_count);
james@685 710 total_remaining -= mdl_byte_count;
james@685 711 }
james@685 712 }
james@685 713 }
james@685 714 if (ScatterGatherBufferLength < FIELD_OFFSET(SCATTER_GATHER_LIST, Elements) +
james@685 715 sizeof(SCATTER_GATHER_ELEMENT) * sglist->NumberOfElements + sizeof(sg_extra_t))
james@685 716 {
james@685 717 //KdPrint((__DRIVER_NAME " STATUS_BUFFER_TOO_SMALL (%d < %d)\n", ScatterGatherBufferLength, FIELD_OFFSET(SCATTER_GATHER_LIST, Elements) +
james@685 718 // sizeof(SCATTER_GATHER_ELEMENT) * sglist->NumberOfElements + sizeof(sg_extra_t)));
james@685 719 return STATUS_BUFFER_TOO_SMALL;
james@685 720 }
james@685 721
james@685 722 sg_extra = (sg_extra_t *)((PUCHAR)sglist + FIELD_OFFSET(SCATTER_GATHER_LIST, Elements) +
james@685 723 (sizeof(SCATTER_GATHER_ELEMENT)) * sglist->NumberOfElements);
james@685 724
james@685 725 sg_extra->allocated_by_me = allocated_by_me;
james@685 726
james@685 727 sg_extra->map_type = map_type;
james@685 728 switch (map_type)
james@685 729 {
james@685 730 case MAP_TYPE_MDL:
james@685 731 //KdPrint((__DRIVER_NAME " MAP_TYPE_MDL - %p\n", MmGetMdlVirtualAddress(Mdl)));
james@685 732 total_remaining = Length;
james@685 733 for (sg_element = 0, curr_mdl = Mdl, active = FALSE; total_remaining > 0; curr_mdl = curr_mdl->Next)
james@685 734 {
james@685 735 ASSERT(curr_mdl);
james@685 736 mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
james@685 737 mdl_byte_count = MmGetMdlByteCount(curr_mdl);
james@685 738 /* need to use <= va + len - 1 to avoid ptr wraparound */
james@685 739 if ((UINT_PTR)CurrentVa >= (UINT_PTR)mdl_start_va && (UINT_PTR)CurrentVa <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
james@685 740 {
james@685 741 active = TRUE;
james@685 742 mdl_byte_count -= (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
james@685 743 mdl_start_va = CurrentVa;
james@685 744 }
james@685 745 if (active && mdl_byte_count)
james@685 746 {
james@685 747 ULONG pfn_offset;
james@685 748 remaining = min(mdl_byte_count, total_remaining);
james@685 749 offset = (ULONG)((UINT_PTR)mdl_start_va & (PAGE_SIZE - 1));
james@685 750 pfn_offset = (ULONG)(((UINT_PTR)mdl_start_va >> PAGE_SHIFT) - ((UINT_PTR)MmGetMdlVirtualAddress(curr_mdl) >> PAGE_SHIFT));
james@685 751 //for (i = 0; i < ADDRESS_AND_SIZE_TO_SPAN_PAGES(mdl_start_va, mdl_byte_count); i++)
james@685 752 for (i = 0; remaining > 0; i++)
james@685 753 {
james@685 754 pfn = MmGetMdlPfnArray(curr_mdl)[pfn_offset + i];
james@685 755 ASSERT(pfn);
james@685 756 gref = (grant_ref_t)GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE, INVALID_GRANT_REF);
james@685 757 ASSERT(gref != INVALID_GRANT_REF);
james@685 758 sglist->Elements[sg_element].Address.QuadPart = (LONGLONG)(gref << PAGE_SHIFT) | offset;
james@685 759 sglist->Elements[sg_element].Length = min(min(PAGE_SIZE - offset, remaining), total_remaining);
james@685 760 total_remaining -= sglist->Elements[sg_element].Length;
james@685 761 remaining -= sglist->Elements[sg_element].Length;
james@685 762 offset = 0;
james@685 763 sg_element++;
james@685 764 }
james@685 765 }
james@685 766 }
james@685 767 if (sg_element != sglist->NumberOfElements)
james@685 768 {
james@685 769 KdPrint((__DRIVER_NAME " sg_element = %d, sglist->NumberOfElements = %d\n", sg_element, sglist->NumberOfElements));
james@685 770 KdPrint((__DRIVER_NAME " CurrentVa = %p, Length = %d\n", CurrentVa, Length));
james@685 771 for (curr_mdl = Mdl; curr_mdl; curr_mdl = curr_mdl->Next)
james@685 772 {
james@685 773 KdPrint((__DRIVER_NAME " Mdl = %p, VirtualAddress = %p, ByteCount = %d\n", Mdl, MmGetMdlVirtualAddress(curr_mdl), MmGetMdlByteCount(curr_mdl)));
james@685 774 }
james@685 775 }
james@685 776 ASSERT(sg_element == sglist->NumberOfElements);
james@685 777 break;
james@685 778 case MAP_TYPE_REMAPPED:
james@685 779 sg_extra->aligned_buffer = ExAllocatePoolWithTag(NonPagedPool, max(remapped_bytes, PAGE_SIZE), XENPCI_POOL_TAG);
james@685 780 if (!sg_extra->aligned_buffer)
james@685 781 {
james@685 782 KdPrint((__DRIVER_NAME " MAP_TYPE_REMAPPED buffer allocation failed - requested va = %p, length = %d\n", MmGetMdlVirtualAddress(Mdl), remapped_bytes));
james@685 783 return STATUS_INSUFFICIENT_RESOURCES;
james@685 784 }
james@685 785 //KdPrint((__DRIVER_NAME " MAP_TYPE_REMAPPED - %p, %d\n", sg_extra->aligned_buffer, remapped_bytes));
james@685 786 //KdPrint((__DRIVER_NAME " CurrentVa = %p, Length = %d\n", CurrentVa, Length));
james@685 787 //for (curr_mdl = Mdl; curr_mdl; curr_mdl = curr_mdl->Next)
james@685 788 //{
james@685 789 // KdPrint((__DRIVER_NAME " Mdl = %p, VirtualAddress = %p, ByteCount = %d\n", Mdl, MmGetMdlVirtualAddress(curr_mdl), MmGetMdlByteCount(curr_mdl)));
james@685 790 //}
james@685 791 sg_extra->mdl = Mdl;
james@685 792 sg_extra->currentva = CurrentVa;
james@685 793 sg_extra->copy_length = remapped_bytes;
james@685 794
james@685 795 if (WriteToDevice)
james@685 796 {
james@685 797 for (curr_mdl = Mdl, offset = 0, active = FALSE; curr_mdl && offset < Length; curr_mdl = curr_mdl->Next)
james@685 798 {
james@685 799 mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
james@685 800 mdl_byte_count = MmGetMdlByteCount(curr_mdl);
james@685 801 mdl_offset = 0;
james@685 802 /* need to use <= va + len - 1 to avoid ptr wraparound */
james@685 803 if ((UINT_PTR)CurrentVa >= (UINT_PTR)mdl_start_va && (UINT_PTR)CurrentVa <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
james@685 804 {
james@685 805 active = TRUE;
james@685 806 mdl_byte_count -= (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
james@685 807 if (offset + mdl_byte_count > Length)
james@685 808 mdl_byte_count = Length - offset;
james@685 809 mdl_offset = (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
james@685 810 mdl_start_va = CurrentVa;
james@685 811 }
james@685 812 if (active)
james@685 813 {
james@685 814 PVOID unaligned_buffer;
james@685 815 unaligned_buffer = (PUCHAR)MmGetSystemAddressForMdlSafe(curr_mdl, NormalPagePriority);
james@685 816 ASSERT(unaligned_buffer); /* lazy */
james@685 817 memcpy((PUCHAR)sg_extra->aligned_buffer + offset, (PUCHAR)unaligned_buffer + mdl_offset, mdl_byte_count);
james@685 818 offset += mdl_byte_count;
james@685 819 }
james@685 820 }
james@685 821 }
james@685 822 for (sg_element = 0, remaining = remapped_bytes;
james@685 823 sg_element < ADDRESS_AND_SIZE_TO_SPAN_PAGES(sg_extra->aligned_buffer, remapped_bytes); sg_element++)
james@685 824 {
james@685 825 pfn = (PFN_NUMBER)(MmGetPhysicalAddress((PUCHAR)sg_extra->aligned_buffer + (sg_element << PAGE_SHIFT)).QuadPart >> PAGE_SHIFT);
james@685 826 ASSERT(pfn);
james@685 827 gref = (grant_ref_t)GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE, INVALID_GRANT_REF);
james@685 828 ASSERT(gref != INVALID_GRANT_REF);
james@685 829 sglist->Elements[sg_element].Address.QuadPart = (ULONGLONG)gref << PAGE_SHIFT;
james@685 830 sglist->Elements[sg_element].Length = min(PAGE_SIZE, remaining);
james@685 831 remaining -= sglist->Elements[sg_element].Length;
james@685 832 }
james@685 833 break;
james@685 834 case MAP_TYPE_VIRTUAL:
james@685 835 ptr = MmGetSystemAddressForMdlSafe(Mdl, NormalPagePriority);
james@685 836 ASSERT(ptr); /* lazy */
james@685 837 sglist->Elements[0].Address.QuadPart = (ULONGLONG)ptr + ((UINT_PTR)CurrentVa - (UINT_PTR)MmGetMdlVirtualAddress(Mdl));
james@685 838 sglist->Elements[0].Length = Length;
james@685 839 //KdPrint((__DRIVER_NAME " MAP_TYPE_VIRTUAL - %08x\n", sglist->Elements[0].Address.LowPart));
james@685 840 break;
james@685 841 default:
james@685 842 KdPrint((__DRIVER_NAME " map_type = %d\n", map_type));
james@685 843 break;
james@685 844 }
james@685 845 //FUNCTION_EXIT();
james@685 846 return STATUS_SUCCESS;
james@685 847 }
james@685 848
james@685 849 static NTSTATUS
james@685 850 XenPci_DOP_BuildScatterGatherList(
james@685 851 IN PDMA_ADAPTER DmaAdapter,
james@685 852 IN PDEVICE_OBJECT DeviceObject,
james@685 853 IN PMDL Mdl,
james@685 854 IN PVOID CurrentVa,
james@685 855 IN ULONG Length,
james@685 856 IN PDRIVER_LIST_CONTROL ExecutionRoutine,
james@685 857 IN PVOID Context,
james@685 858 IN BOOLEAN WriteToDevice,
james@685 859 IN PVOID ScatterGatherBuffer,
james@685 860 IN ULONG ScatterGatherBufferLength)
james@685 861 {
james@685 862 NTSTATUS status;
james@685 863
james@685 864 status = XenPci_DOP_BuildScatterGatherListButDontExecute(DmaAdapter, DeviceObject, Mdl, CurrentVa, Length, WriteToDevice, ScatterGatherBuffer, ScatterGatherBufferLength, FALSE);
james@685 865
james@685 866 if (NT_SUCCESS(status))
james@685 867 ExecutionRoutine(DeviceObject, DeviceObject->CurrentIrp, ScatterGatherBuffer, Context);
james@685 868
james@685 869 //FUNCTION_EXIT();
james@685 870
james@685 871 return status;
james@685 872 }
james@685 873
james@685 874 static NTSTATUS
james@685 875 XenPci_DOP_GetScatterGatherList(
james@685 876 PDMA_ADAPTER DmaAdapter,
james@685 877 PDEVICE_OBJECT DeviceObject,
james@685 878 PMDL Mdl,
james@685 879 PVOID CurrentVa,
james@685 880 ULONG Length,
james@685 881 PDRIVER_LIST_CONTROL ExecutionRoutine,
james@685 882 PVOID Context,
james@685 883 BOOLEAN WriteToDevice)
james@685 884 {
james@685 885 NTSTATUS status;
james@685 886 ULONG list_size;
james@685 887 ULONG map_registers;
james@685 888 PSCATTER_GATHER_LIST sg_list;
james@685 889
james@685 890 //FUNCTION_ENTER();
james@685 891
james@685 892 status = XenPci_DOP_CalculateScatterGatherList(DmaAdapter, Mdl, CurrentVa, Length, &list_size, &map_registers);
james@685 893 if (!NT_SUCCESS(status))
james@685 894 {
james@685 895 //FUNCTION_EXIT();
james@685 896 return status;
james@685 897 }
james@685 898
james@685 899 sg_list = ExAllocatePoolWithTag(NonPagedPool, list_size, XENPCI_POOL_TAG);
james@685 900 if (!sg_list)
james@685 901 {
james@685 902 KdPrint((__DRIVER_NAME " Cannot allocate memory for sg_list\n"));
james@685 903 //FUNCTION_EXIT();
james@685 904 return STATUS_INSUFFICIENT_RESOURCES;
james@685 905 }
james@685 906
james@685 907 status = XenPci_DOP_BuildScatterGatherListButDontExecute(DmaAdapter, DeviceObject, Mdl, CurrentVa, Length, WriteToDevice, sg_list, list_size, TRUE);
james@685 908
james@685 909 if (NT_SUCCESS(status))
james@685 910 ExecutionRoutine(DeviceObject, DeviceObject->CurrentIrp, sg_list, Context);
james@685 911
james@685 912 //FUNCTION_EXIT();
james@685 913
james@685 914 return status;
james@685 915 }
james@685 916
james@685 917 static NTSTATUS
james@685 918 XenPci_DOP_BuildMdlFromScatterGatherList(
james@685 919 PDMA_ADAPTER DmaAdapter,
james@685 920 PSCATTER_GATHER_LIST ScatterGather,
james@685 921 PMDL OriginalMdl,
james@685 922 PMDL *TargetMdl)
james@685 923 {
james@685 924 NTSTATUS status = STATUS_SUCCESS;
james@685 925 UNREFERENCED_PARAMETER(DmaAdapter);
james@685 926 UNREFERENCED_PARAMETER(ScatterGather);
james@685 927 UNREFERENCED_PARAMETER(OriginalMdl);
james@685 928 UNREFERENCED_PARAMETER(TargetMdl);
james@685 929
james@685 930 FUNCTION_ENTER();
james@685 931
james@685 932 if (OriginalMdl)
james@685 933 {
james@685 934 *TargetMdl = OriginalMdl;
james@685 935 }
james@685 936 else
james@685 937 {
james@685 938 *TargetMdl = NULL;
james@685 939 status = STATUS_INVALID_PARAMETER;
james@685 940 }
james@685 941
james@685 942 FUNCTION_EXIT();
james@685 943
james@685 944 return status;
james@685 945 }
james@685 946
james@685 947 PDMA_ADAPTER
james@685 948 XenPci_BIS_GetDmaAdapter(PVOID context, PDEVICE_DESCRIPTION device_description, PULONG number_of_map_registers)
james@685 949 {
james@685 950 xen_dma_adapter_t *xen_dma_adapter;
james@685 951 PDEVICE_OBJECT curr, prev;
james@685 952 PDRIVER_OBJECT fdo_driver_object;
james@685 953 PVOID fdo_driver_extension;
james@685 954
james@685 955 UNREFERENCED_PARAMETER(device_description);
james@685 956
james@685 957 FUNCTION_ENTER();
james@685 958
james@685 959 KdPrint((__DRIVER_NAME " IRQL = %d\n", KeGetCurrentIrql()));
james@685 960 KdPrint((__DRIVER_NAME " Device Description = %p:\n", device_description));
james@685 961 KdPrint((__DRIVER_NAME " Version = %d\n", device_description->Version));
james@685 962 KdPrint((__DRIVER_NAME " Master = %d\n", device_description->Master));
james@685 963 KdPrint((__DRIVER_NAME " ScatterGather = %d\n", device_description->ScatterGather));
james@685 964 KdPrint((__DRIVER_NAME " DemandMode = %d\n", device_description->DemandMode));
james@685 965 KdPrint((__DRIVER_NAME " AutoInitialize = %d\n", device_description->AutoInitialize));
james@685 966 KdPrint((__DRIVER_NAME " Dma32BitAddresses = %d\n", device_description->Dma32BitAddresses));
james@685 967 KdPrint((__DRIVER_NAME " IgnoreCount = %d\n", device_description->IgnoreCount));
james@685 968 KdPrint((__DRIVER_NAME " Dma64BitAddresses = %d\n", device_description->Dma64BitAddresses));
james@685 969 KdPrint((__DRIVER_NAME " BusNumber = %d\n", device_description->BusNumber));
james@685 970 KdPrint((__DRIVER_NAME " DmaChannel = %d\n", device_description->DmaChannel));
james@685 971 KdPrint((__DRIVER_NAME " InterfaceType = %d\n", device_description->InterfaceType));
james@685 972 KdPrint((__DRIVER_NAME " DmaWidth = %d\n", device_description->DmaWidth));
james@685 973 KdPrint((__DRIVER_NAME " DmaSpeed = %d\n", device_description->DmaSpeed));
james@685 974 KdPrint((__DRIVER_NAME " MaximumLength = %d\n", device_description->MaximumLength));
james@685 975 KdPrint((__DRIVER_NAME " DmaPort = %d\n", device_description->DmaPort));
james@685 976
james@685 977 if (!device_description->Master)
james@685 978 return NULL;
james@685 979 /*
james@685 980 we have to allocate PAGE_SIZE bytes here because Windows thinks this is
james@685 981 actually an ADAPTER_OBJECT, and then the verifier crashes because
james@685 982 Windows accessed beyond the end of the structure :(
james@685 983 */
james@685 984 xen_dma_adapter = ExAllocatePoolWithTag(NonPagedPool, PAGE_SIZE, XENPCI_POOL_TAG);
james@685 985 ASSERT(xen_dma_adapter);
james@685 986 RtlZeroMemory(xen_dma_adapter, PAGE_SIZE);
james@685 987
james@685 988 switch(device_description->Version)
james@685 989 {
james@685 990 case DEVICE_DESCRIPTION_VERSION1:
james@685 991 xen_dma_adapter->adapter_object.DmaHeader.Version = 1;
james@685 992 break;
james@685 993 case DEVICE_DESCRIPTION_VERSION: /* ignore what the docs say here - DEVICE_DESCRIPTION_VERSION appears to mean the latest version */
james@685 994 case DEVICE_DESCRIPTION_VERSION2:
james@685 995 xen_dma_adapter->adapter_object.DmaHeader.Version = 2;
james@685 996 break;
james@685 997 default:
james@685 998 KdPrint((__DRIVER_NAME " Unsupported device description version %d\n", device_description->Version));
james@685 999 ExFreePoolWithTag(xen_dma_adapter, XENPCI_POOL_TAG);
james@685 1000 return NULL;
james@685 1001 }
james@685 1002
james@685 1003 xen_dma_adapter->xppdd = context;
james@685 1004 xen_dma_adapter->dma_extension = NULL;
james@685 1005
james@685 1006 KdPrint((__DRIVER_NAME " About to call IoGetAttachedDeviceReference\n"));
james@685 1007 curr = IoGetAttachedDeviceReference(WdfDeviceWdmGetDeviceObject(xen_dma_adapter->xppdd->wdf_device));
james@685 1008 KdPrint((__DRIVER_NAME " Before start of loop - curr = %p\n", curr));
james@685 1009 while (curr != NULL)
james@685 1010 {
james@685 1011 fdo_driver_object = curr->DriverObject;
james@685 1012 if (fdo_driver_object)
james@685 1013 {
james@685 1014 ObReferenceObject(fdo_driver_object);
james@685 1015 fdo_driver_extension = IoGetDriverObjectExtension(fdo_driver_object, UlongToPtr(XEN_DMA_DRIVER_EXTENSION_MAGIC));
james@685 1016 if (fdo_driver_extension)
james@685 1017 {
james@685 1018 xen_dma_adapter->dma_extension_driver = fdo_driver_object; /* so we can dereference it on putdmaadapter */
james@685 1019 xen_dma_adapter->dma_extension = (dma_driver_extension_t *)fdo_driver_extension;
james@685 1020 ObDereferenceObject(curr);
james@685 1021 break;
james@685 1022 }
james@685 1023 else
james@685 1024 {
james@685 1025 ObDereferenceObject(fdo_driver_object);
james@685 1026 }
james@685 1027 }
james@685 1028 prev = curr;
james@685 1029 curr = IoGetLowerDeviceObject(curr);
james@685 1030 ObDereferenceObject(prev);
james@685 1031 }
james@685 1032 KdPrint((__DRIVER_NAME " End of loop\n"));
james@685 1033
james@685 1034 xen_dma_adapter->adapter_object.DmaHeader.Size = sizeof(X_ADAPTER_OBJECT); //xen_dma_adapter_t);
james@685 1035 xen_dma_adapter->adapter_object.MasterAdapter = NULL;
james@685 1036 if (xen_dma_adapter->dma_extension && xen_dma_adapter->dma_extension->max_sg_elements)
james@685 1037 {
james@685 1038 xen_dma_adapter->adapter_object.MapRegistersPerChannel = xen_dma_adapter->dma_extension->max_sg_elements;
james@685 1039 }
james@685 1040 else
james@685 1041 {
james@685 1042 xen_dma_adapter->adapter_object.MapRegistersPerChannel = 256;
james@685 1043 }
james@685 1044 xen_dma_adapter->adapter_object.AdapterBaseVa = NULL;
james@685 1045 xen_dma_adapter->adapter_object.MapRegisterBase = NULL;
james@685 1046 xen_dma_adapter->adapter_object.NumberOfMapRegisters = 0;
james@685 1047 xen_dma_adapter->adapter_object.CommittedMapRegisters = 0;
james@685 1048 xen_dma_adapter->adapter_object.CurrentWcb = NULL;
james@685 1049 KeInitializeDeviceQueue(&xen_dma_adapter->adapter_object.ChannelWaitQueue);
james@685 1050 xen_dma_adapter->adapter_object.RegisterWaitQueue = NULL;
james@685 1051 InitializeListHead(&xen_dma_adapter->adapter_object.AdapterQueue);
james@685 1052 KeInitializeSpinLock(&xen_dma_adapter->adapter_object.SpinLock);
james@685 1053 xen_dma_adapter->adapter_object.MapRegisters = NULL;
james@685 1054 xen_dma_adapter->adapter_object.PagePort = NULL;
james@685 1055 xen_dma_adapter->adapter_object.ChannelNumber = 0xff;
james@685 1056 xen_dma_adapter->adapter_object.AdapterNumber = 0;
james@685 1057 xen_dma_adapter->adapter_object.DmaPortAddress = 0;
james@685 1058 xen_dma_adapter->adapter_object.AdapterMode = 0;
james@685 1059 xen_dma_adapter->adapter_object.NeedsMapRegisters = FALSE; /* when true this causes a crash in the crash dump path */
james@685 1060 xen_dma_adapter->adapter_object.MasterDevice = 1;
james@685 1061 xen_dma_adapter->adapter_object.Width16Bits = 0;
james@685 1062 xen_dma_adapter->adapter_object.ScatterGather = device_description->ScatterGather;
james@685 1063 xen_dma_adapter->adapter_object.IgnoreCount = device_description->IgnoreCount;
james@685 1064 xen_dma_adapter->adapter_object.Dma32BitAddresses = device_description->Dma32BitAddresses;
james@685 1065 xen_dma_adapter->adapter_object.Dma64BitAddresses = device_description->Dma64BitAddresses;
james@685 1066 InitializeListHead(&xen_dma_adapter->adapter_object.AdapterList);
james@685 1067
james@685 1068 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations = ExAllocatePoolWithTag(NonPagedPool, sizeof(DMA_OPERATIONS), XENPCI_POOL_TAG);
james@685 1069 ASSERT(xen_dma_adapter->adapter_object.DmaHeader.DmaOperations);
james@685 1070 if (xen_dma_adapter->adapter_object.DmaHeader.Version == 1)
james@685 1071 {
james@685 1072 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->Size = FIELD_OFFSET(DMA_OPERATIONS, CalculateScatterGatherList);
james@685 1073 }
james@685 1074 else
james@685 1075 {
james@685 1076 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->Size = sizeof(DMA_OPERATIONS);
james@685 1077 }
james@685 1078 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->PutDmaAdapter = XenPci_DOP_PutDmaAdapter;
james@685 1079 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->AllocateCommonBuffer = XenPci_DOP_AllocateCommonBuffer;
james@685 1080 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->FreeCommonBuffer = XenPci_DOP_FreeCommonBuffer;
james@685 1081 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->AllocateAdapterChannel = XenPci_DOP_AllocateAdapterChannel;
james@685 1082 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->FlushAdapterBuffers = XenPci_DOP_FlushAdapterBuffers;
james@685 1083 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->FreeAdapterChannel = XenPci_DOP_FreeAdapterChannel;
james@685 1084 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->FreeMapRegisters = XenPci_DOP_FreeMapRegisters;
james@685 1085 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->MapTransfer = XenPci_DOP_MapTransfer;
james@685 1086 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->GetDmaAlignment = XenPci_DOP_GetDmaAlignment;
james@685 1087 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->ReadDmaCounter = XenPci_DOP_ReadDmaCounter;
james@685 1088 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->GetScatterGatherList = XenPci_DOP_GetScatterGatherList;
james@685 1089 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->PutScatterGatherList = XenPci_DOP_PutScatterGatherList;
james@685 1090 if (xen_dma_adapter->adapter_object.DmaHeader.Version == 2)
james@685 1091 {
james@685 1092 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->CalculateScatterGatherList = XenPci_DOP_CalculateScatterGatherList;
james@685 1093 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->BuildScatterGatherList = XenPci_DOP_BuildScatterGatherList;
james@685 1094 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->BuildMdlFromScatterGatherList = XenPci_DOP_BuildMdlFromScatterGatherList;
james@685 1095 }
james@685 1096
james@685 1097 *number_of_map_registers = xen_dma_adapter->adapter_object.MapRegistersPerChannel; //1024; /* why not... */
james@685 1098
james@685 1099 FUNCTION_EXIT();
james@685 1100
james@685 1101 return &xen_dma_adapter->adapter_object.DmaHeader;
james@685 1102 }
james@685 1103
james@685 1104 ULONG
james@685 1105 XenPci_BIS_SetBusData(PVOID context, ULONG data_type, PVOID buffer, ULONG offset, ULONG length)
james@685 1106 {
james@685 1107 UNREFERENCED_PARAMETER(context);
james@685 1108 UNREFERENCED_PARAMETER(data_type);
james@685 1109 UNREFERENCED_PARAMETER(buffer);
james@685 1110 UNREFERENCED_PARAMETER(offset);
james@685 1111 UNREFERENCED_PARAMETER(length);
james@685 1112
james@685 1113 FUNCTION_ENTER();
james@685 1114 FUNCTION_EXIT();
james@685 1115 return 0;
james@685 1116 }
james@685 1117
james@685 1118 ULONG
james@685 1119 XenPci_BIS_GetBusData(PVOID context, ULONG data_type, PVOID buffer, ULONG offset, ULONG length)
james@685 1120 {
james@685 1121 UNREFERENCED_PARAMETER(context);
james@685 1122 UNREFERENCED_PARAMETER(data_type);
james@685 1123 UNREFERENCED_PARAMETER(buffer);
james@685 1124 UNREFERENCED_PARAMETER(offset);
james@685 1125 UNREFERENCED_PARAMETER(length);
james@685 1126
james@685 1127 FUNCTION_ENTER();
james@685 1128 FUNCTION_EXIT();
james@685 1129 return 0;
james@685 1130 }