win-pvdrivers

view xenpci/xenpci_dma.c @ 766:6300617040e0

Big changes - not ready for production use.
Removed all the custom DMA handling code as it was completely incompatible with the Windows verifier.
Added liblfds (using the lock free stack) from liblfds.org so that grant's can be obtained at DIRQL.
Fixed xennet and xenvbd to support the changes.
xenusb and xenscsi almost certainly will not yet work after the changes.
author James Harper <james.harper@bendigoit.com.au>
date Sun Jan 31 21:28:42 2010 +1100 (2010-01-31)
parents eeb1a1d80778
children
line source
1 /*
2 PV Drivers for Windows Xen HVM Domains
3 Copyright (C) 2009 James Harper
5 This program is free software; you can redistribute it and/or
6 modify it under the terms of the GNU General Public License
7 as published by the Free Software Foundation; either version 2
8 of the License, or (at your option) any later version.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 */
20 #include "xenpci.h"
21 #include <stdlib.h>
22 #include <io/ring.h>
24 #pragma warning(disable : 4200) // zero-sized array
25 #pragma warning(disable: 4127) // conditional expression is constant
27 #define MAP_TYPE_INVALID 0
28 #define MAP_TYPE_VIRTUAL 1
29 #define MAP_TYPE_MDL 2
30 #define MAP_TYPE_REMAPPED 3
33 #if 0
34 kd> dt _ADAPTER_OBJECT 81e96b08 -v
35 hal!_ADAPTER_OBJECT
36 struct _ADAPTER_OBJECT, 26 elements, 0x64 bytes
37 +0x000 DmaHeader : struct _DMA_ADAPTER, 3 elements, 0x8 bytes
38 +0x008 MasterAdapter : (null)
39 +0x00c MapRegistersPerChannel : 0x80001
40 +0x010 AdapterBaseVa : (null)
41 +0x014 MapRegisterBase : (null)
42 +0x018 NumberOfMapRegisters : 0
43 +0x01c CommittedMapRegisters : 0
44 +0x020 CurrentWcb : (null)
45 +0x024 ChannelWaitQueue : struct _KDEVICE_QUEUE, 5 elements, 0x14 bytes
46 +0x038 RegisterWaitQueue : (null)
47 +0x03c AdapterQueue : struct _LIST_ENTRY, 2 elements, 0x8 bytes
48 [ 0x0 - 0x0 ]
49 +0x044 SpinLock : 0
50 +0x048 MapRegisters : (null)
51 +0x04c PagePort : (null)
52 +0x050 ChannelNumber : 0xff ''
53 +0x051 AdapterNumber : 0 ''
54 +0x052 DmaPortAddress : 0
55 +0x054 AdapterMode : 0 ''
56 +0x055 NeedsMapRegisters : 0 ''
57 +0x056 MasterDevice : 0x1 ''
58 +0x057 Width16Bits : 0 ''
59 +0x058 ScatterGather : 0x1 ''
60 +0x059 IgnoreCount : 0 ''
61 +0x05a Dma32BitAddresses : 0x1 ''
62 +0x05b Dma64BitAddresses : 0 ''
63 +0x05c AdapterList : struct _LIST_ENTRY, 2 elements, 0x8 bytes
64 [ 0x806e1250 - 0x81f1b474 ]
65 #endif
67 /* need to confirm that this is the same for AMD64 too */
68 typedef struct {
69 DMA_ADAPTER DmaHeader;
70 PVOID MasterAdapter;
71 ULONG MapRegistersPerChannel;
72 PVOID AdapterBaseVa;
73 PVOID MapRegisterBase;
74 ULONG NumberOfMapRegisters;
75 ULONG CommittedMapRegisters;
76 PVOID CurrentWcb;
77 KDEVICE_QUEUE ChannelWaitQueue;
78 PKDEVICE_QUEUE RegisterWaitQueue;
79 LIST_ENTRY AdapterQueue;
80 KSPIN_LOCK SpinLock;
81 PVOID MapRegisters;
82 PVOID PagePort;
83 UCHAR ChannelNumber;
84 UCHAR AdapterNumber;
85 USHORT DmaPortAddress;
86 UCHAR AdapterMode;
87 BOOLEAN NeedsMapRegisters;
88 BOOLEAN MasterDevice;
89 UCHAR Width16Bits;
90 BOOLEAN ScatterGather;
91 BOOLEAN IgnoreCount;
92 BOOLEAN Dma32BitAddresses;
93 BOOLEAN Dma64BitAddresses;
94 #if (NTDDI_VERSION >= NTDDI_WS03)
95 BOOLEAN LegacyAdapter;
96 #endif
97 LIST_ENTRY AdapterList;
98 } X_ADAPTER_OBJECT;
100 typedef struct {
101 ULONG map_type;
102 PVOID aligned_buffer;
103 ULONG copy_length;
104 PMDL mdl;
105 PVOID currentva;
106 BOOLEAN allocated_by_me;
107 } sg_extra_t;
109 typedef struct {
110 ULONG map_type;
111 PVOID aligned_buffer;
112 PVOID unaligned_buffer;
113 ULONG copy_length;
114 grant_ref_t gref;
115 PHYSICAL_ADDRESS logical;
116 } map_register_t;
118 typedef struct {
119 PDEVICE_OBJECT device_object;
120 ULONG total_map_registers;
121 PDRIVER_CONTROL execution_routine;
122 PIRP current_irp;
123 PVOID context;
124 ULONG count;
125 map_register_t regs[1];
126 } map_register_base_t;
128 typedef struct {
129 X_ADAPTER_OBJECT adapter_object;
130 PXENPCI_PDO_DEVICE_DATA xppdd;
131 dma_driver_extension_t *dma_extension;
132 PDRIVER_OBJECT dma_extension_driver; /* to deference it */
133 map_register_base_t *map_register_base;
134 map_register_base_t *queued_map_register_base;
135 KSPIN_LOCK lock;
136 } xen_dma_adapter_t;
138 BOOLEAN
139 XenPci_BIS_TranslateBusAddress(PVOID context, PHYSICAL_ADDRESS bus_address, ULONG length, PULONG address_space, PPHYSICAL_ADDRESS translated_address)
140 {
141 UNREFERENCED_PARAMETER(context);
142 UNREFERENCED_PARAMETER(length);
143 /* actually this isn't right - should look up the gref for the physical address and work backwards from that */
144 FUNCTION_ENTER();
145 if (*address_space != 0)
146 {
147 KdPrint((__DRIVER_NAME " Cannot map I/O space\n"));
148 FUNCTION_EXIT();
149 return FALSE;
150 }
151 *translated_address = bus_address;
152 FUNCTION_EXIT();
153 return TRUE;
154 }
156 static VOID
157 XenPci_DOP_PutDmaAdapter(PDMA_ADAPTER dma_adapter)
158 {
159 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
161 FUNCTION_ENTER();
163 ASSERT(!xen_dma_adapter->map_register_base);
164 ASSERT(!xen_dma_adapter->queued_map_register_base);
165 if (xen_dma_adapter->dma_extension)
166 ObDereferenceObject(xen_dma_adapter->dma_extension_driver);
167 ExFreePoolWithTag(xen_dma_adapter->adapter_object.DmaHeader.DmaOperations, XENPCI_POOL_TAG);
168 ExFreePoolWithTag(xen_dma_adapter, XENPCI_POOL_TAG);
170 FUNCTION_EXIT();
172 return;
173 }
175 static PVOID
176 XenPci_DOP_AllocateCommonBuffer(
177 PDMA_ADAPTER DmaAdapter,
178 ULONG Length,
179 PPHYSICAL_ADDRESS LogicalAddress,
180 BOOLEAN CacheEnabled
181 )
182 {
183 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)DmaAdapter;
184 PXENPCI_DEVICE_DATA xpdd;
185 PVOID buffer;
186 PFN_NUMBER pfn;
187 grant_ref_t gref;
189 UNREFERENCED_PARAMETER(CacheEnabled);
191 FUNCTION_ENTER();
192 //KdPrint((__DRIVER_NAME " Length = %d\n", Length));
194 xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
196 buffer = ExAllocatePoolWithTag(NonPagedPool, Length, XENPCI_POOL_TAG);
197 //KdPrint((__DRIVER_NAME " buffer = %p\n", buffer));
199 ASSERT(buffer); /* lazy */
201 pfn = (PFN_NUMBER)(MmGetPhysicalAddress(buffer).QuadPart >> PAGE_SHIFT);
202 //KdPrint((__DRIVER_NAME " pfn = %08x\n", (ULONG)pfn));
203 ASSERT(pfn); /* lazy */
205 gref = (grant_ref_t)GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE, INVALID_GRANT_REF);
206 //KdPrint((__DRIVER_NAME " gref = %08x\n", (ULONG)gref));
207 ASSERT(gref != INVALID_GRANT_REF); /* lazy */
209 LogicalAddress->QuadPart = (gref << PAGE_SHIFT) | (PtrToUlong(buffer) & (PAGE_SIZE - 1));
210 KdPrint((__DRIVER_NAME " logical = %08x%08x\n", LogicalAddress->HighPart, LogicalAddress->LowPart));
212 FUNCTION_EXIT();
214 return buffer;
215 }
217 static VOID
218 XenPci_DOP_FreeCommonBuffer(
219 PDMA_ADAPTER dma_adapter,
220 ULONG length,
221 PHYSICAL_ADDRESS logical_address,
222 PVOID virtual_address,
223 BOOLEAN cache_enabled
224 )
225 {
226 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
227 PXENPCI_DEVICE_DATA xpdd;
228 grant_ref_t gref;
230 UNREFERENCED_PARAMETER(dma_adapter);
231 UNREFERENCED_PARAMETER(length);
232 UNREFERENCED_PARAMETER(cache_enabled);
234 // FUNCTION_ENTER();
236 xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
237 gref = (grant_ref_t)(logical_address.QuadPart >> PAGE_SHIFT);
238 //KdPrint((__DRIVER_NAME " F Releasing Grant Ref %d\n", gref));
239 GntTbl_EndAccess(xpdd, gref, FALSE);
240 //KdPrint((__DRIVER_NAME " F Released Grant Ref\n"));
241 ExFreePoolWithTag(virtual_address, XENPCI_POOL_TAG);
243 // FUNCTION_EXIT();
244 }
246 static VOID
247 XenPci_ExecuteMapRegisterDma(
248 PDMA_ADAPTER dma_adapter,
249 map_register_base_t *map_register_base)
250 {
251 IO_ALLOCATION_ACTION action;
253 UNREFERENCED_PARAMETER(dma_adapter);
255 action = map_register_base->execution_routine(map_register_base->device_object, map_register_base->current_irp, map_register_base, map_register_base->context);
257 switch (action)
258 {
259 case KeepObject:
260 KdPrint((__DRIVER_NAME " KeepObject\n"));
261 ASSERT(FALSE);
262 break;
263 case DeallocateObject:
264 KdPrint((__DRIVER_NAME " DeallocateObject\n"));
265 ASSERT(FALSE);
266 break;
267 case DeallocateObjectKeepRegisters:
268 //KdPrint((__DRIVER_NAME " DeallocateObjectKeepRegisters\n"));
269 break;
270 default:
271 KdPrint((__DRIVER_NAME " Unknown action %d\n", action));
272 ASSERT(FALSE);
273 break;
274 }
275 return;
276 }
278 static NTSTATUS
279 XenPci_DOP_AllocateAdapterChannel(
280 IN PDMA_ADAPTER dma_adapter,
281 IN PDEVICE_OBJECT device_object,
282 IN ULONG NumberOfMapRegisters,
283 IN PDRIVER_CONTROL ExecutionRoutine,
284 IN PVOID Context
285 )
286 {
287 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
288 PXENPCI_DEVICE_DATA xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
289 ULONG i;
290 map_register_base_t *map_register_base;
292 UNREFERENCED_PARAMETER(dma_adapter);
294 //FUNCTION_ENTER();
296 ASSERT(!xen_dma_adapter->queued_map_register_base);
298 map_register_base = ExAllocatePoolWithTag(NonPagedPool,
299 FIELD_OFFSET(map_register_base_t, regs) + NumberOfMapRegisters * sizeof(map_register_t), XENPCI_POOL_TAG);
300 if (!map_register_base)
301 {
302 KdPrint((__DRIVER_NAME " Cannot allocate memory for map_register_base\n"));
303 //FUNCTION_EXIT();
304 return STATUS_INSUFFICIENT_RESOURCES;
305 }
306 //KdPrint((__DRIVER_NAME " Alloc %p\n", map_register_base));
307 /* we should also allocate a single page of memory here for remap purposes as once we allocate the map registers there is no failure allowed */
308 map_register_base->device_object = device_object;
309 map_register_base->current_irp = device_object->CurrentIrp;
310 map_register_base->total_map_registers = NumberOfMapRegisters;
311 map_register_base->execution_routine = ExecutionRoutine;
312 map_register_base->context = Context;
313 map_register_base->count = 0;
315 for (i = 0; i < NumberOfMapRegisters; i++)
316 {
317 map_register_base->regs[i].gref = GntTbl_GetRef(xpdd);
318 if (map_register_base->regs[i].gref == INVALID_GRANT_REF)
319 {
320 KdPrint((__DRIVER_NAME " Not enough gref's for AdapterChannel list\n"));
321 /* go back through the list and free the ones we allocated */
322 NumberOfMapRegisters = i;
323 for (i = 0; i < NumberOfMapRegisters; i++)
324 {
325 GntTbl_PutRef(xpdd, map_register_base->regs[i].gref);
326 }
327 //KdPrint((__DRIVER_NAME " B Free %p\n", map_register_base));
328 ExFreePoolWithTag(map_register_base, XENPCI_POOL_TAG);
329 return STATUS_INSUFFICIENT_RESOURCES;
330 }
331 map_register_base->regs[i].map_type = MAP_TYPE_INVALID;
332 map_register_base->regs[i].aligned_buffer = NULL;
333 map_register_base->regs[i].unaligned_buffer = NULL;
334 map_register_base->regs[i].copy_length = 0;
335 }
337 KeAcquireSpinLockAtDpcLevel(&xen_dma_adapter->lock);
338 if (xen_dma_adapter->map_register_base)
339 {
340 xen_dma_adapter->queued_map_register_base = map_register_base;
341 KeReleaseSpinLockFromDpcLevel(&xen_dma_adapter->lock);
342 }
343 else
344 {
345 xen_dma_adapter->map_register_base = map_register_base;
346 KeReleaseSpinLockFromDpcLevel(&xen_dma_adapter->lock);
347 XenPci_ExecuteMapRegisterDma(dma_adapter, map_register_base);
348 }
350 //FUNCTION_EXIT();
351 return STATUS_SUCCESS;
352 }
354 static BOOLEAN
355 XenPci_DOP_FlushAdapterBuffers(
356 PDMA_ADAPTER dma_adapter,
357 PMDL mdl,
358 PVOID MapRegisterBase,
359 PVOID CurrentVa,
360 ULONG Length,
361 BOOLEAN write_to_device)
362 {
363 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
364 //PXENPCI_DEVICE_DATA xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
365 map_register_base_t *map_register_base = MapRegisterBase;
366 map_register_t *map_register;
367 ULONG i;
369 UNREFERENCED_PARAMETER(xen_dma_adapter);
370 UNREFERENCED_PARAMETER(mdl);
371 UNREFERENCED_PARAMETER(CurrentVa);
372 UNREFERENCED_PARAMETER(Length);
374 //FUNCTION_ENTER();
376 ASSERT(xen_dma_adapter->map_register_base);
377 ASSERT(xen_dma_adapter->map_register_base == map_register_base);
379 for (i = 0; i < map_register_base->count; i++)
380 {
381 map_register = &map_register_base->regs[i];
382 if (map_register->map_type == MAP_TYPE_REMAPPED && !write_to_device)
383 memcpy(map_register->unaligned_buffer, map_register->aligned_buffer, map_register->copy_length);
384 }
385 //FUNCTION_EXIT();
387 return TRUE;
388 }
390 static VOID
391 XenPci_DOP_FreeAdapterChannel(
392 IN PDMA_ADAPTER DmaAdapter
393 )
394 {
395 UNREFERENCED_PARAMETER(DmaAdapter);
397 FUNCTION_ENTER();
398 FUNCTION_EXIT();
399 }
401 static VOID
402 XenPci_DOP_FreeMapRegisters(
403 PDMA_ADAPTER dma_adapter,
404 PVOID MapRegisterBase,
405 ULONG NumberOfMapRegisters)
406 {
407 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
408 PXENPCI_DEVICE_DATA xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
409 map_register_base_t *map_register_base = MapRegisterBase;
410 map_register_t *map_register;
411 ULONG i;
413 UNREFERENCED_PARAMETER(NumberOfMapRegisters);
415 //FUNCTION_ENTER();
417 if (!map_register_base)
418 {
419 /* i'm not sure if this is ideal here, but NDIS definitely does it */
420 return;
421 }
423 ASSERT(xen_dma_adapter->map_register_base == map_register_base);
424 ASSERT(map_register_base->total_map_registers == NumberOfMapRegisters);
426 for (i = 0; i < map_register_base->total_map_registers; i++)
427 {
428 map_register = &map_register_base->regs[i];
429 GntTbl_EndAccess(xpdd, map_register->gref, FALSE);
430 switch (map_register->map_type)
431 {
432 case MAP_TYPE_INVALID:
433 break;
434 case MAP_TYPE_REMAPPED:
435 ExFreePoolWithTag(map_register->aligned_buffer, XENPCI_POOL_TAG);
436 break;
437 case MAP_TYPE_MDL:
438 break;
439 case MAP_TYPE_VIRTUAL:
440 break;
441 }
442 }
443 ExFreePoolWithTag(map_register_base, XENPCI_POOL_TAG);
445 KeAcquireSpinLockAtDpcLevel(&xen_dma_adapter->lock);
446 if (xen_dma_adapter->queued_map_register_base)
447 {
448 xen_dma_adapter->map_register_base = xen_dma_adapter->queued_map_register_base;
449 xen_dma_adapter->queued_map_register_base = NULL;
450 KeReleaseSpinLockFromDpcLevel(&xen_dma_adapter->lock);
451 XenPci_ExecuteMapRegisterDma(dma_adapter, xen_dma_adapter->map_register_base);
452 }
453 else
454 {
455 xen_dma_adapter->map_register_base = NULL;
456 KeReleaseSpinLockFromDpcLevel(&xen_dma_adapter->lock);
457 }
459 //FUNCTION_EXIT();
460 }
462 static PHYSICAL_ADDRESS
463 XenPci_DOP_MapTransfer(
464 PDMA_ADAPTER dma_adapter,
465 PMDL mdl,
466 PVOID MapRegisterBase,
467 PVOID CurrentVa,
468 PULONG Length,
469 BOOLEAN WriteToDevice)
470 {
471 xen_dma_adapter_t *xen_dma_adapter = (xen_dma_adapter_t *)dma_adapter;
472 PXENPCI_DEVICE_DATA xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
473 map_register_base_t *map_register_base = MapRegisterBase;
474 map_register_t *map_register = &map_register_base->regs[map_register_base->count];
475 PDEVICE_OBJECT device_object = map_register_base->device_object;
476 ULONG page_offset;
477 PFN_NUMBER pfn;
478 //grant_ref_t gref;
479 PUCHAR ptr;
480 ULONG mdl_offset;
481 ULONG pfn_index;
483 //FUNCTION_ENTER();
485 //KdPrint((__DRIVER_NAME " Mdl = %p, MapRegisterBase = %p, MdlVa = %p, CurrentVa = %p, Length = %d\n",
486 // mdl, MapRegisterBase, MmGetMdlVirtualAddress(mdl), CurrentVa, *Length));
488 ASSERT(mdl);
489 ASSERT(map_register_base);
490 ASSERT(xen_dma_adapter->map_register_base == map_register_base);
491 ASSERT(map_register_base->count < map_register_base->total_map_registers);
493 if (xen_dma_adapter->dma_extension)
494 {
495 if (xen_dma_adapter->dma_extension->need_virtual_address && xen_dma_adapter->dma_extension->need_virtual_address(device_object->CurrentIrp))
496 {
497 map_register->map_type = MAP_TYPE_VIRTUAL;
498 }
499 else
500 {
501 if (xen_dma_adapter->dma_extension->get_alignment)
502 {
503 ULONG alignment = xen_dma_adapter->dma_extension->get_alignment(device_object->CurrentIrp);
504 if ((MmGetMdlByteOffset(mdl) & (alignment - 1)) || (MmGetMdlByteCount(mdl) & (alignment - 1)))
505 {
506 map_register->map_type = MAP_TYPE_REMAPPED;
507 }
508 else
509 {
510 map_register->map_type = MAP_TYPE_MDL;
511 }
512 }
513 else
514 {
515 map_register->map_type = MAP_TYPE_MDL;
516 }
517 }
518 }
519 else
520 {
521 map_register->map_type = MAP_TYPE_MDL;
522 }
524 switch (map_register->map_type)
525 {
526 case MAP_TYPE_MDL:
527 //KdPrint((__DRIVER_NAME " MAP_TYPE_MDL\n"));
528 mdl_offset = (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)MmGetMdlVirtualAddress(mdl));
529 page_offset = PtrToUlong(CurrentVa) & (PAGE_SIZE - 1);
530 *Length = min(*Length, PAGE_SIZE - page_offset);
531 pfn_index = (ULONG)(((UINT_PTR)CurrentVa >> PAGE_SHIFT) - ((UINT_PTR)MmGetMdlVirtualAddress(mdl) >> PAGE_SHIFT));
532 //KdPrint((__DRIVER_NAME " mdl_offset = %d, page_offset = %d, length = %d, pfn_index = %d\n",
533 // mdl_offset, page_offset, *Length, pfn_index));
534 pfn = MmGetMdlPfnArray(mdl)[pfn_index];
535 //KdPrint((__DRIVER_NAME " B Requesting Grant Ref\n"));
537 //ASSERT(map_register->gref != INVALID_GRANT_REF);
538 GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE, map_register->gref);
539 //KdPrint((__DRIVER_NAME " B Got Grant Ref %d\n", gref));
540 map_register->logical.QuadPart = ((LONGLONG)map_register->gref << PAGE_SHIFT) | page_offset;
541 map_register_base->count++;
542 break;
543 case MAP_TYPE_REMAPPED:
544 //KdPrint((__DRIVER_NAME " MAP_TYPE_REMAPPED (MapTransfer)\n"));
545 //KdPrint((__DRIVER_NAME " Mdl = %p, MapRegisterBase = %p, MdlVa = %p, CurrentVa = %p, Length = %d\n",
546 // mdl, MapRegisterBase, MmGetMdlVirtualAddress(mdl), CurrentVa, *Length));
547 mdl_offset = (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)MmGetMdlVirtualAddress(mdl));
548 *Length = min(*Length, PAGE_SIZE);
549 map_register->aligned_buffer = ExAllocatePoolWithTag(NonPagedPool, PAGE_SIZE, XENPCI_POOL_TAG);
550 ASSERT(map_register->aligned_buffer);
551 map_register->unaligned_buffer = MmGetSystemAddressForMdlSafe(mdl, NormalPagePriority);
552 ASSERT(map_register->unaligned_buffer); /* lazy */
553 map_register->unaligned_buffer = (PUCHAR)map_register->unaligned_buffer + mdl_offset;
554 map_register->copy_length = *Length;
555 if (WriteToDevice)
556 memcpy(map_register->aligned_buffer, map_register->unaligned_buffer, map_register->copy_length);
557 pfn = (PFN_NUMBER)(MmGetPhysicalAddress(map_register->aligned_buffer).QuadPart >> PAGE_SHIFT);
558 //KdPrint((__DRIVER_NAME " C Requesting Grant Ref\n"));
559 GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE, map_register->gref);
560 //KdPrint((__DRIVER_NAME " C Got Grant Ref %d\n", gref));
561 map_register->logical.QuadPart = ((LONGLONG)map_register->gref << PAGE_SHIFT);
562 map_register_base->count++;
563 break;
564 case MAP_TYPE_VIRTUAL:
565 //KdPrint((__DRIVER_NAME " MAP_TYPE_VIRTUAL\n"));
566 ptr = MmGetSystemAddressForMdlSafe(mdl, NormalPagePriority);
567 ASSERT(ptr); /* lazy */
568 map_register->logical.QuadPart = (ULONGLONG)ptr;
569 map_register_base->count++;
570 break;
571 default:
572 ASSERT(FALSE);
573 break;
574 }
576 //KdPrint((__DRIVER_NAME " logical = %08x:%08x\n", map_register->logical.HighPart, map_register->logical.LowPart));
577 //FUNCTION_EXIT();
578 return map_register->logical;
579 }
581 static ULONG
582 XenPci_DOP_GetDmaAlignment(
583 PDMA_ADAPTER DmaAdapter)
584 {
585 UNREFERENCED_PARAMETER(DmaAdapter);
587 FUNCTION_ENTER();
588 FUNCTION_EXIT();
589 return 0;
590 }
592 static ULONG
593 XenPci_DOP_ReadDmaCounter(
594 PDMA_ADAPTER DmaAdapter)
595 {
596 UNREFERENCED_PARAMETER(DmaAdapter);
598 FUNCTION_ENTER();
599 FUNCTION_EXIT();
600 return 0;
601 }
603 static VOID
604 XenPci_DOP_PutScatterGatherList(
605 IN PDMA_ADAPTER DmaAdapter,
606 IN PSCATTER_GATHER_LIST sg_list,
607 IN BOOLEAN WriteToDevice
608 )
609 {
610 xen_dma_adapter_t *xen_dma_adapter;
611 PXENPCI_DEVICE_DATA xpdd;
612 ULONG i;
613 sg_extra_t *sg_extra;
614 PMDL curr_mdl;
615 ULONG offset;
616 BOOLEAN active;
618 UNREFERENCED_PARAMETER(WriteToDevice);
620 //FUNCTION_ENTER();
622 xen_dma_adapter = (xen_dma_adapter_t *)DmaAdapter;
623 ASSERT(xen_dma_adapter);
624 xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
626 sg_extra = (sg_extra_t *)((PUCHAR)sg_list + FIELD_OFFSET(SCATTER_GATHER_LIST, Elements) +
627 (sizeof(SCATTER_GATHER_ELEMENT)) * sg_list->NumberOfElements);
629 switch (sg_extra->map_type)
630 {
631 case MAP_TYPE_REMAPPED:
632 for (i = 0; i < sg_list->NumberOfElements; i++)
633 {
634 grant_ref_t gref;
635 gref = (grant_ref_t)(sg_list->Elements[i].Address.QuadPart >> PAGE_SHIFT);
636 GntTbl_EndAccess(xpdd, gref, FALSE);
637 sg_list->Elements[i].Address.QuadPart = -1;
638 }
639 ASSERT(sg_extra->mdl);
640 if (!WriteToDevice)
641 {
642 for (curr_mdl = sg_extra->mdl, offset = 0, active = FALSE; curr_mdl && offset < sg_extra->copy_length; curr_mdl = curr_mdl->Next)
643 {
644 PVOID mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
645 ULONG mdl_byte_count = MmGetMdlByteCount(curr_mdl);
646 ULONG mdl_offset = 0;
647 /* need to use <= va + len - 1 to avoid ptr wraparound */
648 if ((UINT_PTR)sg_extra->currentva >= (UINT_PTR)mdl_start_va && (UINT_PTR)sg_extra->currentva <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
649 {
650 active = TRUE;
651 mdl_byte_count -= (ULONG)((UINT_PTR)sg_extra->currentva - (UINT_PTR)mdl_start_va);
652 if (offset + mdl_byte_count > sg_extra->copy_length)
653 mdl_byte_count = sg_extra->copy_length - offset;
654 mdl_offset = (ULONG)((UINT_PTR)sg_extra->currentva - (UINT_PTR)mdl_start_va);
655 mdl_start_va = sg_extra->currentva;
656 }
657 if (active)
658 {
659 PVOID unaligned_buffer;
660 unaligned_buffer = MmGetSystemAddressForMdlSafe(curr_mdl, NormalPagePriority);
661 ASSERT(unaligned_buffer); /* lazy */
662 memcpy((PUCHAR)unaligned_buffer + mdl_offset, (PUCHAR)sg_extra->aligned_buffer + offset, mdl_byte_count);
663 offset += mdl_byte_count;
664 }
665 }
666 ASSERT(offset == sg_extra->copy_length);
667 }
668 ExFreePoolWithTag(sg_extra->aligned_buffer, XENPCI_POOL_TAG);
669 break;
670 case MAP_TYPE_MDL:
671 for (i = 0; i < sg_list->NumberOfElements; i++)
672 {
673 grant_ref_t gref;
674 gref = (grant_ref_t)(sg_list->Elements[i].Address.QuadPart >> PAGE_SHIFT);
675 GntTbl_EndAccess(xpdd, gref, FALSE);
676 sg_list->Elements[i].Address.QuadPart = -1;
677 }
678 break;
679 case MAP_TYPE_VIRTUAL:
680 break;
681 }
682 if (sg_extra->allocated_by_me)
683 ExFreePoolWithTag(sg_list, XENPCI_POOL_TAG);
684 //FUNCTION_EXIT();
685 }
687 static NTSTATUS
688 XenPci_DOP_CalculateScatterGatherList(
689 PDMA_ADAPTER DmaAdapter,
690 PMDL Mdl,
691 PVOID CurrentVa,
692 ULONG Length,
693 PULONG ScatterGatherListSize,
694 PULONG NumberOfMapRegisters
695 )
696 {
697 xen_dma_adapter_t *xen_dma_adapter;
698 ULONG elements;
699 PMDL curr_mdl;
701 UNREFERENCED_PARAMETER(CurrentVa);
703 //FUNCTION_ENTER();
705 //KdPrint((__DRIVER_NAME " Mdl = %p\n", Mdl));
706 //KdPrint((__DRIVER_NAME " CurrentVa = %p\n", CurrentVa));
707 //KdPrint((__DRIVER_NAME " Length = %d\n", Length));
709 xen_dma_adapter = (xen_dma_adapter_t *)DmaAdapter;
711 if (Mdl)
712 {
713 //if (CurrentVa != MmGetMdlVirtualAddress(Mdl))
714 //{
715 // KdPrint((__DRIVER_NAME " CurrentVa (%p) != MdlVa (%p)\n", CurrentVa, MmGetMdlVirtualAddress(Mdl)));
716 //
718 //KdPrint((__DRIVER_NAME " CurrentVa = %p, MdlVa = %p\n", CurrentVa, MmGetMdlVirtualAddress(Mdl)));
720 for (curr_mdl = Mdl, elements = 0; curr_mdl; curr_mdl = curr_mdl->Next)
721 {
722 //KdPrint((__DRIVER_NAME " curr_mdlVa = %p, curr_mdl size = %d\n", MmGetMdlVirtualAddress(curr_mdl), MmGetMdlByteCount(curr_mdl)));
723 elements += ADDRESS_AND_SIZE_TO_SPAN_PAGES(MmGetMdlVirtualAddress(curr_mdl), MmGetMdlByteCount(curr_mdl));
724 }
725 }
726 else
727 {
728 elements = ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, Length); // + 1;
729 }
731 if (elements > xen_dma_adapter->adapter_object.MapRegistersPerChannel)
732 {
733 //KdPrint((__DRIVER_NAME " elements = %d - too many\n", elements));
734 if (NumberOfMapRegisters)
735 *NumberOfMapRegisters = 0;
736 *ScatterGatherListSize = 0;
738 return STATUS_INSUFFICIENT_RESOURCES;
739 }
741 *ScatterGatherListSize = FIELD_OFFSET(SCATTER_GATHER_LIST, Elements)
742 + sizeof(SCATTER_GATHER_ELEMENT) * elements
743 + sizeof(sg_extra_t);
744 if (NumberOfMapRegisters)
745 *NumberOfMapRegisters = elements;
747 //KdPrint((__DRIVER_NAME " ScatterGatherListSize = %d, NumberOfMapRegisters = %d\n", *ScatterGatherListSize, elements));
749 //FUNCTION_EXIT();
750 return STATUS_SUCCESS;
751 }
753 static NTSTATUS
754 XenPci_DOP_BuildScatterGatherListButDontExecute(
755 IN PDMA_ADAPTER DmaAdapter,
756 IN PDEVICE_OBJECT DeviceObject,
757 IN PMDL Mdl,
758 IN PVOID CurrentVa,
759 IN ULONG Length,
760 IN BOOLEAN WriteToDevice,
761 IN PVOID ScatterGatherBuffer,
762 IN ULONG ScatterGatherBufferLength,
763 BOOLEAN allocated_by_me)
764 {
765 ULONG i;
766 PSCATTER_GATHER_LIST sglist = ScatterGatherBuffer;
767 PUCHAR ptr;
768 ULONG remaining = Length;
769 ULONG total_remaining;
770 xen_dma_adapter_t *xen_dma_adapter;
771 PXENPCI_DEVICE_DATA xpdd;
772 sg_extra_t *sg_extra;
773 PMDL curr_mdl;
774 ULONG map_type;
775 ULONG sg_element;
776 ULONG offset;
777 PFN_NUMBER pfn;
778 grant_ref_t gref;
779 BOOLEAN active;
780 PVOID mdl_start_va;
781 ULONG mdl_byte_count;
782 ULONG mdl_offset;
783 ULONG remapped_bytes = 0;
785 //FUNCTION_ENTER();
787 if (!ScatterGatherBuffer)
788 {
789 KdPrint((__DRIVER_NAME " NULL ScatterGatherBuffer\n"));
790 return STATUS_INVALID_PARAMETER;
791 }
792 //if (MmGetMdlVirtualAddress(Mdl) != CurrentVa)
793 //{
794 // KdPrint((__DRIVER_NAME " MmGetMdlVirtualAddress = %p, CurrentVa = %p, Length = %d\n", MmGetMdlVirtualAddress(Mdl), CurrentVa, Length));
795 //}
797 xen_dma_adapter = (xen_dma_adapter_t *)DmaAdapter;
798 xpdd = GetXpdd(xen_dma_adapter->xppdd->wdf_device_bus_fdo);
800 ASSERT(Mdl);
802 if (xen_dma_adapter->dma_extension)
803 {
804 if (xen_dma_adapter->dma_extension->need_virtual_address && xen_dma_adapter->dma_extension->need_virtual_address(DeviceObject->CurrentIrp))
805 {
806 ASSERT(!Mdl->Next); /* can only virtual a single buffer */
807 //ASSERT(MmGetMdlVirtualAddress(Mdl) == CurrentVa);
808 map_type = MAP_TYPE_VIRTUAL;
809 sglist->NumberOfElements = 1;
810 }
811 else
812 {
813 if (xen_dma_adapter->dma_extension->get_alignment)
814 {
815 ULONG alignment = xen_dma_adapter->dma_extension->get_alignment(DeviceObject->CurrentIrp);
817 map_type = MAP_TYPE_MDL;
818 sglist->NumberOfElements = 0;
819 for (curr_mdl = Mdl, remapped_bytes = 0, active = FALSE; remapped_bytes < Length && curr_mdl; curr_mdl = curr_mdl->Next)
820 {
821 mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
822 mdl_byte_count = MmGetMdlByteCount(curr_mdl);
823 /* need to use <= va + len - 1 to avoid ptr wraparound */
824 if ((UINT_PTR)CurrentVa >= (UINT_PTR)mdl_start_va && (UINT_PTR)CurrentVa <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
825 {
826 active = TRUE;
827 mdl_byte_count -= (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
828 if (remapped_bytes + mdl_byte_count > Length)
829 mdl_byte_count = Length - remapped_bytes;
830 mdl_start_va = CurrentVa;
831 }
832 if (active)
833 {
834 if (((UINT_PTR)mdl_start_va & (alignment - 1)) || (mdl_byte_count & (alignment - 1)))
835 map_type = MAP_TYPE_REMAPPED;
836 remapped_bytes += mdl_byte_count;
837 if (remapped_bytes > Length)
838 remapped_bytes = Length;
839 }
840 }
841 if (remapped_bytes != Length)
842 {
843 KdPrint((__DRIVER_NAME " remapped_bytes = %d, Length = %d\n", remapped_bytes, Length));
844 }
845 //ASSERT(remapped_bytes == Length);
846 sglist->NumberOfElements += ADDRESS_AND_SIZE_TO_SPAN_PAGES(NULL, remapped_bytes);
847 }
848 else
849 {
850 map_type = MAP_TYPE_MDL;
851 }
852 }
853 }
854 else
855 {
856 map_type = MAP_TYPE_MDL;
857 }
858 if (map_type == MAP_TYPE_MDL)
859 {
860 for (curr_mdl = Mdl, sglist->NumberOfElements = 0, total_remaining = Length, active = FALSE; total_remaining > 0; curr_mdl = curr_mdl->Next)
861 {
862 ASSERT(curr_mdl);
863 mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
864 mdl_byte_count = MmGetMdlByteCount(curr_mdl);
865 /* need to use <= va + len - 1 to avoid ptr wraparound */
866 if ((UINT_PTR)CurrentVa >= (UINT_PTR)mdl_start_va && (UINT_PTR)CurrentVa <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
867 {
868 active = TRUE;
869 mdl_byte_count -= (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
870 mdl_start_va = CurrentVa;
871 }
872 mdl_byte_count = min(mdl_byte_count, total_remaining);
873 if (active && mdl_byte_count)
874 {
875 sglist->NumberOfElements += ADDRESS_AND_SIZE_TO_SPAN_PAGES(
876 mdl_start_va, mdl_byte_count);
877 total_remaining -= mdl_byte_count;
878 }
879 }
880 }
881 if (ScatterGatherBufferLength < FIELD_OFFSET(SCATTER_GATHER_LIST, Elements) +
882 sizeof(SCATTER_GATHER_ELEMENT) * sglist->NumberOfElements + sizeof(sg_extra_t))
883 {
884 //KdPrint((__DRIVER_NAME " STATUS_BUFFER_TOO_SMALL (%d < %d)\n", ScatterGatherBufferLength, FIELD_OFFSET(SCATTER_GATHER_LIST, Elements) +
885 // sizeof(SCATTER_GATHER_ELEMENT) * sglist->NumberOfElements + sizeof(sg_extra_t)));
886 return STATUS_BUFFER_TOO_SMALL;
887 }
889 if (map_type != MAP_TYPE_VIRTUAL)
890 {
891 for (sg_element = 0; sg_element < sglist->NumberOfElements; sg_element++)
892 {
893 gref = GntTbl_GetRef(xpdd);
894 if (gref == INVALID_GRANT_REF)
895 {
896 KdPrint((__DRIVER_NAME " Not enough gref's for SG list\n"));
897 /* go back through the list and free the ones we allocated */
898 sglist->NumberOfElements = sg_element;
899 for (sg_element = 0; sg_element < sglist->NumberOfElements; sg_element++)
900 {
901 gref = (grant_ref_t)(sglist->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT);
902 GntTbl_PutRef(xpdd, gref);
903 }
904 return STATUS_INSUFFICIENT_RESOURCES;
905 }
906 sglist->Elements[sg_element].Address.QuadPart = ((LONGLONG)gref << PAGE_SHIFT);
907 }
908 }
910 sg_extra = (sg_extra_t *)((PUCHAR)sglist + FIELD_OFFSET(SCATTER_GATHER_LIST, Elements) +
911 (sizeof(SCATTER_GATHER_ELEMENT)) * sglist->NumberOfElements);
913 sg_extra->allocated_by_me = allocated_by_me;
915 sg_extra->map_type = map_type;
916 switch (map_type)
917 {
918 case MAP_TYPE_MDL:
919 //KdPrint((__DRIVER_NAME " MAP_TYPE_MDL - %p\n", MmGetMdlVirtualAddress(Mdl)));
920 total_remaining = Length;
921 for (sg_element = 0, curr_mdl = Mdl, active = FALSE; total_remaining > 0; curr_mdl = curr_mdl->Next)
922 {
923 ASSERT(curr_mdl);
924 mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
925 mdl_byte_count = MmGetMdlByteCount(curr_mdl);
926 /* need to use <= va + len - 1 to avoid ptr wraparound */
927 if ((UINT_PTR)CurrentVa >= (UINT_PTR)mdl_start_va && (UINT_PTR)CurrentVa <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
928 {
929 active = TRUE;
930 mdl_byte_count -= (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
931 mdl_start_va = CurrentVa;
932 }
933 if (active && mdl_byte_count)
934 {
935 ULONG pfn_offset;
936 remaining = min(mdl_byte_count, total_remaining);
937 offset = (ULONG)((UINT_PTR)mdl_start_va & (PAGE_SIZE - 1));
938 pfn_offset = (ULONG)(((UINT_PTR)mdl_start_va >> PAGE_SHIFT) - ((UINT_PTR)MmGetMdlVirtualAddress(curr_mdl) >> PAGE_SHIFT));
939 //for (i = 0; i < ADDRESS_AND_SIZE_TO_SPAN_PAGES(mdl_start_va, mdl_byte_count); i++)
940 for (i = 0; remaining > 0; i++)
941 {
942 pfn = MmGetMdlPfnArray(curr_mdl)[pfn_offset + i];
943 //ASSERT((grant_ref_t)(sglist->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT) != INVALID_GRANT_REF);
944 if ((grant_ref_t)(sglist->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT) == INVALID_GRANT_REF)
945 KdPrint((__DRIVER_NAME " GGG\n"));
947 GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE,
948 (grant_ref_t)(sglist->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT));
949 sglist->Elements[sg_element].Address.QuadPart |= (LONGLONG)offset;
950 sglist->Elements[sg_element].Length = min(min(PAGE_SIZE - offset, remaining), total_remaining);
951 total_remaining -= sglist->Elements[sg_element].Length;
952 remaining -= sglist->Elements[sg_element].Length;
953 offset = 0;
954 sg_element++;
955 }
956 }
957 }
958 if (sg_element != sglist->NumberOfElements)
959 {
960 KdPrint((__DRIVER_NAME " sg_element = %d, sglist->NumberOfElements = %d\n", sg_element, sglist->NumberOfElements));
961 KdPrint((__DRIVER_NAME " CurrentVa = %p, Length = %d\n", CurrentVa, Length));
962 for (curr_mdl = Mdl; curr_mdl; curr_mdl = curr_mdl->Next)
963 {
964 KdPrint((__DRIVER_NAME " Mdl = %p, VirtualAddress = %p, ByteCount = %d\n", Mdl, MmGetMdlVirtualAddress(curr_mdl), MmGetMdlByteCount(curr_mdl)));
965 }
966 }
967 ASSERT(sg_element == sglist->NumberOfElements);
968 break;
969 case MAP_TYPE_REMAPPED:
970 sg_extra->aligned_buffer = ExAllocatePoolWithTag(NonPagedPool, max(remapped_bytes, PAGE_SIZE), XENPCI_POOL_TAG);
971 if (!sg_extra->aligned_buffer)
972 {
973 KdPrint((__DRIVER_NAME " MAP_TYPE_REMAPPED buffer allocation failed - requested va = %p, length = %d\n", MmGetMdlVirtualAddress(Mdl), remapped_bytes));
974 return STATUS_INSUFFICIENT_RESOURCES;
975 }
976 //KdPrint((__DRIVER_NAME " MAP_TYPE_REMAPPED - %p, %d\n", sg_extra->aligned_buffer, remapped_bytes));
977 //KdPrint((__DRIVER_NAME " CurrentVa = %p, Length = %d\n", CurrentVa, Length));
978 //for (curr_mdl = Mdl; curr_mdl; curr_mdl = curr_mdl->Next)
979 //{
980 // KdPrint((__DRIVER_NAME " Mdl = %p, VirtualAddress = %p, ByteCount = %d\n", Mdl, MmGetMdlVirtualAddress(curr_mdl), MmGetMdlByteCount(curr_mdl)));
981 //}
982 sg_extra->mdl = Mdl;
983 sg_extra->currentva = CurrentVa;
984 sg_extra->copy_length = remapped_bytes;
986 if (WriteToDevice)
987 {
988 for (curr_mdl = Mdl, offset = 0, active = FALSE; curr_mdl && offset < Length; curr_mdl = curr_mdl->Next)
989 {
990 mdl_start_va = MmGetMdlVirtualAddress(curr_mdl);
991 mdl_byte_count = MmGetMdlByteCount(curr_mdl);
992 mdl_offset = 0;
993 /* need to use <= va + len - 1 to avoid ptr wraparound */
994 if ((UINT_PTR)CurrentVa >= (UINT_PTR)mdl_start_va && (UINT_PTR)CurrentVa <= (UINT_PTR)mdl_start_va + mdl_byte_count - 1)
995 {
996 active = TRUE;
997 mdl_byte_count -= (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
998 if (offset + mdl_byte_count > Length)
999 mdl_byte_count = Length - offset;
1000 mdl_offset = (ULONG)((UINT_PTR)CurrentVa - (UINT_PTR)mdl_start_va);
1001 mdl_start_va = CurrentVa;
1003 if (active)
1005 PVOID unaligned_buffer;
1006 unaligned_buffer = (PUCHAR)MmGetSystemAddressForMdlSafe(curr_mdl, NormalPagePriority);
1007 ASSERT(unaligned_buffer); /* lazy */
1008 memcpy((PUCHAR)sg_extra->aligned_buffer + offset, (PUCHAR)unaligned_buffer + mdl_offset, mdl_byte_count);
1009 offset += mdl_byte_count;
1013 for (sg_element = 0, remaining = remapped_bytes;
1014 sg_element < ADDRESS_AND_SIZE_TO_SPAN_PAGES(sg_extra->aligned_buffer, remapped_bytes); sg_element++)
1016 pfn = (PFN_NUMBER)(MmGetPhysicalAddress((PUCHAR)sg_extra->aligned_buffer + (sg_element << PAGE_SHIFT)).QuadPart >> PAGE_SHIFT);
1017 //ASSERT((grant_ref_t)(sglist->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT) != INVALID_GRANT_REF);
1018 if ((grant_ref_t)(sglist->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT) == INVALID_GRANT_REF)
1019 KdPrint((__DRIVER_NAME " HHH\n"));
1020 GntTbl_GrantAccess(xpdd, 0, (ULONG)pfn, FALSE,
1021 (grant_ref_t)(sglist->Elements[sg_element].Address.QuadPart >> PAGE_SHIFT));
1022 sglist->Elements[sg_element].Length = min(PAGE_SIZE, remaining);
1023 remaining -= sglist->Elements[sg_element].Length;
1025 break;
1026 case MAP_TYPE_VIRTUAL:
1027 ptr = MmGetSystemAddressForMdlSafe(Mdl, NormalPagePriority);
1028 ASSERT(ptr); /* lazy */
1029 sglist->Elements[0].Address.QuadPart = (ULONGLONG)ptr + ((UINT_PTR)CurrentVa - (UINT_PTR)MmGetMdlVirtualAddress(Mdl));
1030 sglist->Elements[0].Length = Length;
1031 //KdPrint((__DRIVER_NAME " MAP_TYPE_VIRTUAL - %08x\n", sglist->Elements[0].Address.LowPart));
1032 break;
1033 default:
1034 KdPrint((__DRIVER_NAME " map_type = %d\n", map_type));
1035 break;
1037 //FUNCTION_EXIT();
1038 return STATUS_SUCCESS;
1041 static NTSTATUS
1042 XenPci_DOP_BuildScatterGatherList(
1043 IN PDMA_ADAPTER DmaAdapter,
1044 IN PDEVICE_OBJECT DeviceObject,
1045 IN PMDL Mdl,
1046 IN PVOID CurrentVa,
1047 IN ULONG Length,
1048 IN PDRIVER_LIST_CONTROL ExecutionRoutine,
1049 IN PVOID Context,
1050 IN BOOLEAN WriteToDevice,
1051 IN PVOID ScatterGatherBuffer,
1052 IN ULONG ScatterGatherBufferLength)
1054 NTSTATUS status;
1056 status = XenPci_DOP_BuildScatterGatherListButDontExecute(DmaAdapter, DeviceObject, Mdl, CurrentVa, Length, WriteToDevice, ScatterGatherBuffer, ScatterGatherBufferLength, FALSE);
1058 if (NT_SUCCESS(status))
1059 ExecutionRoutine(DeviceObject, DeviceObject->CurrentIrp, ScatterGatherBuffer, Context);
1061 //FUNCTION_EXIT();
1063 return status;
1066 static NTSTATUS
1067 XenPci_DOP_GetScatterGatherList(
1068 PDMA_ADAPTER DmaAdapter,
1069 PDEVICE_OBJECT DeviceObject,
1070 PMDL Mdl,
1071 PVOID CurrentVa,
1072 ULONG Length,
1073 PDRIVER_LIST_CONTROL ExecutionRoutine,
1074 PVOID Context,
1075 BOOLEAN WriteToDevice)
1077 NTSTATUS status;
1078 ULONG list_size;
1079 ULONG map_registers;
1080 PSCATTER_GATHER_LIST sg_list;
1082 //FUNCTION_ENTER();
1084 status = XenPci_DOP_CalculateScatterGatherList(DmaAdapter, Mdl, CurrentVa, Length, &list_size, &map_registers);
1085 if (!NT_SUCCESS(status))
1087 //FUNCTION_EXIT();
1088 return status;
1091 sg_list = ExAllocatePoolWithTag(NonPagedPool, list_size, XENPCI_POOL_TAG);
1092 if (!sg_list)
1094 KdPrint((__DRIVER_NAME " Cannot allocate memory for sg_list\n"));
1095 //FUNCTION_EXIT();
1096 return STATUS_INSUFFICIENT_RESOURCES;
1099 status = XenPci_DOP_BuildScatterGatherListButDontExecute(DmaAdapter, DeviceObject, Mdl, CurrentVa, Length, WriteToDevice, sg_list, list_size, TRUE);
1101 if (NT_SUCCESS(status))
1103 /* sg_list is free'd via PutScatterGatherList later */
1104 ExecutionRoutine(DeviceObject, DeviceObject->CurrentIrp, sg_list, Context);
1106 else
1107 ExFreePoolWithTag(sg_list, XENPCI_POOL_TAG);
1109 //FUNCTION_EXIT();
1111 return status;
1114 static NTSTATUS
1115 XenPci_DOP_BuildMdlFromScatterGatherList(
1116 PDMA_ADAPTER DmaAdapter,
1117 PSCATTER_GATHER_LIST ScatterGather,
1118 PMDL OriginalMdl,
1119 PMDL *TargetMdl)
1121 NTSTATUS status = STATUS_SUCCESS;
1122 UNREFERENCED_PARAMETER(DmaAdapter);
1123 UNREFERENCED_PARAMETER(ScatterGather);
1124 UNREFERENCED_PARAMETER(OriginalMdl);
1125 UNREFERENCED_PARAMETER(TargetMdl);
1127 FUNCTION_ENTER();
1129 if (OriginalMdl)
1131 *TargetMdl = OriginalMdl;
1133 else
1135 *TargetMdl = NULL;
1136 status = STATUS_INVALID_PARAMETER;
1139 FUNCTION_EXIT();
1141 return status;
1144 PDMA_ADAPTER
1145 XenPci_BIS_GetDmaAdapter(PVOID context, PDEVICE_DESCRIPTION device_description, PULONG number_of_map_registers)
1147 xen_dma_adapter_t *xen_dma_adapter;
1148 PDEVICE_OBJECT curr, prev;
1149 PDRIVER_OBJECT fdo_driver_object;
1150 PVOID fdo_driver_extension;
1152 UNREFERENCED_PARAMETER(device_description);
1154 FUNCTION_ENTER();
1156 KdPrint((__DRIVER_NAME " IRQL = %d\n", KeGetCurrentIrql()));
1157 KdPrint((__DRIVER_NAME " Device Description = %p:\n", device_description));
1158 KdPrint((__DRIVER_NAME " Version = %d\n", device_description->Version));
1159 KdPrint((__DRIVER_NAME " Master = %d\n", device_description->Master));
1160 KdPrint((__DRIVER_NAME " ScatterGather = %d\n", device_description->ScatterGather));
1161 KdPrint((__DRIVER_NAME " DemandMode = %d\n", device_description->DemandMode));
1162 KdPrint((__DRIVER_NAME " AutoInitialize = %d\n", device_description->AutoInitialize));
1163 KdPrint((__DRIVER_NAME " Dma32BitAddresses = %d\n", device_description->Dma32BitAddresses));
1164 KdPrint((__DRIVER_NAME " IgnoreCount = %d\n", device_description->IgnoreCount));
1165 KdPrint((__DRIVER_NAME " Dma64BitAddresses = %d\n", device_description->Dma64BitAddresses));
1166 KdPrint((__DRIVER_NAME " BusNumber = %d\n", device_description->BusNumber));
1167 KdPrint((__DRIVER_NAME " DmaChannel = %d\n", device_description->DmaChannel));
1168 KdPrint((__DRIVER_NAME " InterfaceType = %d\n", device_description->InterfaceType));
1169 KdPrint((__DRIVER_NAME " DmaWidth = %d\n", device_description->DmaWidth));
1170 KdPrint((__DRIVER_NAME " DmaSpeed = %d\n", device_description->DmaSpeed));
1171 KdPrint((__DRIVER_NAME " MaximumLength = %d\n", device_description->MaximumLength));
1172 KdPrint((__DRIVER_NAME " DmaPort = %d\n", device_description->DmaPort));
1174 if (!device_description->Master)
1175 return NULL;
1176 /*
1177 we have to allocate PAGE_SIZE bytes here because Windows thinks this is
1178 actually an ADAPTER_OBJECT, and then the verifier crashes because
1179 Windows accessed beyond the end of the structure :(
1180 */
1181 xen_dma_adapter = ExAllocatePoolWithTag(NonPagedPool, PAGE_SIZE, XENPCI_POOL_TAG);
1182 ASSERT(xen_dma_adapter);
1183 RtlZeroMemory(xen_dma_adapter, PAGE_SIZE);
1185 switch(device_description->Version)
1187 case DEVICE_DESCRIPTION_VERSION1:
1188 xen_dma_adapter->adapter_object.DmaHeader.Version = 1;
1189 break;
1190 case DEVICE_DESCRIPTION_VERSION: /* ignore what the docs say here - DEVICE_DESCRIPTION_VERSION appears to mean the latest version */
1191 case DEVICE_DESCRIPTION_VERSION2:
1192 xen_dma_adapter->adapter_object.DmaHeader.Version = 2;
1193 break;
1194 default:
1195 KdPrint((__DRIVER_NAME " Unsupported device description version %d\n", device_description->Version));
1196 ExFreePoolWithTag(xen_dma_adapter, XENPCI_POOL_TAG);
1197 return NULL;
1200 xen_dma_adapter->xppdd = context;
1201 xen_dma_adapter->dma_extension = NULL;
1203 KdPrint((__DRIVER_NAME " About to call IoGetAttachedDeviceReference\n"));
1204 curr = IoGetAttachedDeviceReference(WdfDeviceWdmGetDeviceObject(xen_dma_adapter->xppdd->wdf_device));
1205 KdPrint((__DRIVER_NAME " Before start of loop - curr = %p\n", curr));
1206 while (curr != NULL)
1208 fdo_driver_object = curr->DriverObject;
1209 if (fdo_driver_object)
1211 ObReferenceObject(fdo_driver_object);
1212 fdo_driver_extension = IoGetDriverObjectExtension(fdo_driver_object, UlongToPtr(XEN_DMA_DRIVER_EXTENSION_MAGIC));
1213 if (fdo_driver_extension)
1215 xen_dma_adapter->dma_extension_driver = fdo_driver_object; /* so we can dereference it on putdmaadapter */
1216 xen_dma_adapter->dma_extension = (dma_driver_extension_t *)fdo_driver_extension;
1217 ObDereferenceObject(curr);
1218 break;
1220 else
1222 ObDereferenceObject(fdo_driver_object);
1225 prev = curr;
1226 curr = IoGetLowerDeviceObject(curr);
1227 ObDereferenceObject(prev);
1229 KdPrint((__DRIVER_NAME " End of loop\n"));
1231 xen_dma_adapter->adapter_object.DmaHeader.Size = sizeof(DMA_ADAPTER); //sizeof(X_ADAPTER_OBJECT); //xen_dma_adapter_t);
1232 xen_dma_adapter->adapter_object.MasterAdapter = NULL;
1233 if (xen_dma_adapter->dma_extension && xen_dma_adapter->dma_extension->max_sg_elements)
1235 xen_dma_adapter->adapter_object.MapRegistersPerChannel = xen_dma_adapter->dma_extension->max_sg_elements;
1237 else
1239 xen_dma_adapter->adapter_object.MapRegistersPerChannel = 256;
1241 xen_dma_adapter->adapter_object.AdapterBaseVa = NULL;
1242 xen_dma_adapter->adapter_object.MapRegisterBase = NULL;
1243 xen_dma_adapter->adapter_object.NumberOfMapRegisters = 0;
1244 xen_dma_adapter->adapter_object.CommittedMapRegisters = 0;
1245 xen_dma_adapter->adapter_object.CurrentWcb = NULL;
1246 KeInitializeDeviceQueue(&xen_dma_adapter->adapter_object.ChannelWaitQueue);
1247 xen_dma_adapter->adapter_object.RegisterWaitQueue = NULL;
1248 InitializeListHead(&xen_dma_adapter->adapter_object.AdapterQueue);
1249 KeInitializeSpinLock(&xen_dma_adapter->adapter_object.SpinLock);
1250 xen_dma_adapter->adapter_object.MapRegisters = NULL;
1251 xen_dma_adapter->adapter_object.PagePort = NULL;
1252 xen_dma_adapter->adapter_object.ChannelNumber = 0xff;
1253 xen_dma_adapter->adapter_object.AdapterNumber = 0;
1254 xen_dma_adapter->adapter_object.DmaPortAddress = 0;
1255 xen_dma_adapter->adapter_object.AdapterMode = 0;
1256 xen_dma_adapter->adapter_object.NeedsMapRegisters = FALSE; /* when true this causes a crash in the crash dump path */
1257 xen_dma_adapter->adapter_object.MasterDevice = 1;
1258 xen_dma_adapter->adapter_object.Width16Bits = 0;
1259 xen_dma_adapter->adapter_object.ScatterGather = device_description->ScatterGather;
1260 xen_dma_adapter->adapter_object.IgnoreCount = device_description->IgnoreCount;
1261 xen_dma_adapter->adapter_object.Dma32BitAddresses = device_description->Dma32BitAddresses;
1262 xen_dma_adapter->adapter_object.Dma64BitAddresses = device_description->Dma64BitAddresses;
1263 InitializeListHead(&xen_dma_adapter->adapter_object.AdapterList);
1265 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations = ExAllocatePoolWithTag(NonPagedPool, sizeof(DMA_OPERATIONS), XENPCI_POOL_TAG);
1266 ASSERT(xen_dma_adapter->adapter_object.DmaHeader.DmaOperations);
1267 if (xen_dma_adapter->adapter_object.DmaHeader.Version == 1)
1269 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->Size = FIELD_OFFSET(DMA_OPERATIONS, CalculateScatterGatherList);
1271 else
1273 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->Size = sizeof(DMA_OPERATIONS);
1275 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->PutDmaAdapter = XenPci_DOP_PutDmaAdapter;
1276 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->AllocateCommonBuffer = XenPci_DOP_AllocateCommonBuffer;
1277 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->FreeCommonBuffer = XenPci_DOP_FreeCommonBuffer;
1278 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->AllocateAdapterChannel = XenPci_DOP_AllocateAdapterChannel;
1279 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->FlushAdapterBuffers = XenPci_DOP_FlushAdapterBuffers;
1280 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->FreeAdapterChannel = XenPci_DOP_FreeAdapterChannel;
1281 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->FreeMapRegisters = XenPci_DOP_FreeMapRegisters;
1282 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->MapTransfer = XenPci_DOP_MapTransfer;
1283 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->GetDmaAlignment = XenPci_DOP_GetDmaAlignment;
1284 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->ReadDmaCounter = XenPci_DOP_ReadDmaCounter;
1285 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->GetScatterGatherList = XenPci_DOP_GetScatterGatherList;
1286 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->PutScatterGatherList = XenPci_DOP_PutScatterGatherList;
1287 if (xen_dma_adapter->adapter_object.DmaHeader.Version == 2)
1289 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->CalculateScatterGatherList = XenPci_DOP_CalculateScatterGatherList;
1290 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->BuildScatterGatherList = XenPci_DOP_BuildScatterGatherList;
1291 xen_dma_adapter->adapter_object.DmaHeader.DmaOperations->BuildMdlFromScatterGatherList = XenPci_DOP_BuildMdlFromScatterGatherList;
1294 *number_of_map_registers = xen_dma_adapter->adapter_object.MapRegistersPerChannel; //1024; /* why not... */
1296 KeInitializeSpinLock(&xen_dma_adapter->lock);
1297 xen_dma_adapter->map_register_base = NULL;
1298 xen_dma_adapter->queued_map_register_base = NULL;
1300 FUNCTION_EXIT();
1302 if (KD_DEBUGGER_ENABLED)
1303 KdBreakPoint();
1305 return &xen_dma_adapter->adapter_object.DmaHeader;
1308 ULONG
1309 XenPci_BIS_SetBusData(PVOID context, ULONG data_type, PVOID buffer, ULONG offset, ULONG length)
1311 UNREFERENCED_PARAMETER(context);
1312 UNREFERENCED_PARAMETER(data_type);
1313 UNREFERENCED_PARAMETER(buffer);
1314 UNREFERENCED_PARAMETER(offset);
1315 UNREFERENCED_PARAMETER(length);
1317 FUNCTION_ENTER();
1318 FUNCTION_EXIT();
1319 return 0;
1322 ULONG
1323 XenPci_BIS_GetBusData(PVOID context, ULONG data_type, PVOID buffer, ULONG offset, ULONG length)
1325 UNREFERENCED_PARAMETER(context);
1326 UNREFERENCED_PARAMETER(data_type);
1327 UNREFERENCED_PARAMETER(buffer);
1328 UNREFERENCED_PARAMETER(offset);
1329 UNREFERENCED_PARAMETER(length);
1331 FUNCTION_ENTER();
1332 FUNCTION_EXIT();
1333 return 0;