win-pvdrivers

view xenpci/evtchn.c @ 1106:2d392ecdd366

Fix race is xenvbd causing 30 second freeze under high load
author James Harper <james.harper@bendigoit.com.au>
date Tue Nov 11 23:08:11 2014 +1100 (2014-11-11)
parents 27bd2a5a4704
children
line source
1 /*
2 PV Drivers for Windows Xen HVM Domains
4 Copyright (c) 2014, James Harper
5 All rights reserved.
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9 * Redistributions of source code must retain the above copyright
10 notice, this list of conditions and the following disclaimer.
11 * Redistributions in binary form must reproduce the above copyright
12 notice, this list of conditions and the following disclaimer in the
13 documentation and/or other materials provided with the distribution.
14 * Neither the name of James Harper nor the
15 names of its contributors may be used to endorse or promote products
16 derived from this software without specific prior written permission.
18 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 DISCLAIMED. IN NO EVENT SHALL JAMES HARPER BE LIABLE FOR ANY
22 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
30 #include "xenpci.h"
32 /* Not really necessary but keeps PREfast happy */
33 #if (VER_PRODUCTBUILD >= 7600)
34 static KDEFERRED_ROUTINE EvtChn_DpcBounce;
35 #endif
37 #if defined(_X86_)
38 #define xchg(p1, p2) InterlockedExchange(p1, p2)
39 #define synch_clear_bit(p1, p2) InterlockedBitTestAndReset(p2, p1)
40 #define synch_set_bit(p1, p2) InterlockedBitTestAndSet(p2, p1)
41 #define bit_scan_forward(p1, p2) _BitScanForward(p1, p2)
42 #else
43 #define xchg(p1, p2) InterlockedExchange64(p1, p2)
44 #define synch_clear_bit(p1, p2) InterlockedBitTestAndReset64(p2, p1)
45 #define synch_set_bit(p1, p2) InterlockedBitTestAndSet64(p2, p1)
46 #define bit_scan_forward(p1, p2) _BitScanForward64(p1, p2)
47 #endif
49 #define BITS_PER_LONG (sizeof(xen_ulong_t) * 8)
50 #define BITS_PER_LONG_SHIFT (5 + (sizeof(xen_ulong_t) >> 3))
52 static VOID
53 EvtChn_DpcBounce(PRKDPC Dpc, PVOID Context, PVOID SystemArgument1, PVOID SystemArgument2) {
54 ev_action_t *action = Context;
56 UNREFERENCED_PARAMETER(Dpc);
57 UNREFERENCED_PARAMETER(SystemArgument1);
58 UNREFERENCED_PARAMETER(SystemArgument2);
60 //FUNCTION_ENTER();
62 if (action->type != EVT_ACTION_TYPE_EMPTY) {
63 action->ServiceRoutine(action->ServiceContext);
64 }
65 //FUNCTION_EXIT();
66 }
68 #if 0
69 /* Called at DIRQL */
70 BOOLEAN
71 EvtChn_AckEvent(PVOID context, evtchn_port_t port, BOOLEAN *last_interrupt) {
72 PXENPCI_DEVICE_DATA xpdd = context;
73 ULONG pcpu = KeGetCurrentProcessorNumber() & 0xff;
74 ULONG evt_word;
75 ULONG evt_bit;
76 xen_ulong_t val;
77 int i;
79 evt_bit = port & (BITS_PER_LONG - 1);
80 evt_word = port >> BITS_PER_LONG_SHIFT;
82 val = synch_clear_bit(evt_bit, (volatile xen_long_t *)&xpdd->evtchn_pending_pvt[pcpu][evt_word]);
83 *last_interrupt = TRUE;
84 for (i = 0; i < ARRAY_SIZE(xpdd->evtchn_pending_pvt[pcpu]); i++)
85 {
86 if (xpdd->evtchn_pending_pvt[pcpu][i])
87 {
88 *last_interrupt = FALSE;
89 break;
90 }
91 }
93 return (BOOLEAN)!!val;
94 }
95 #endif
97 //volatile ULONG in_inq = 0;
99 BOOLEAN
100 EvtChn_EvtInterruptIsr(WDFINTERRUPT interrupt, ULONG message_id)
101 {
102 /*
103 For HVM domains, Xen always triggers the event on CPU0. Because the
104 interrupt is delivered via the virtual PCI device it might get delivered
105 to CPU != 0, but we should always use vcpu_info[0]
106 */
107 int vcpu = 0;
108 ULONG pcpu = KeGetCurrentProcessorNumber() & 0xff;
109 vcpu_info_t *vcpu_info;
110 PXENPCI_DEVICE_DATA xpdd = GetXpdd(WdfInterruptGetDevice(interrupt));
111 shared_info_t *shared_info_area = xpdd->shared_info_area;
112 xen_ulong_t evt_words;
113 unsigned long evt_word;
114 unsigned long evt_bit;
115 unsigned int port;
116 ev_action_t *ev_action;
117 BOOLEAN handled = FALSE;
118 BOOLEAN deferred = FALSE;
119 int i;
121 UNREFERENCED_PARAMETER(message_id);
123 if (xpdd->interrupts_masked) {
124 FUNCTION_MSG("unhandled interrupt\n");
125 }
127 if (xpdd->hibernated) {
128 FUNCTION_MSG("interrupt while hibernated\n");
129 }
131 for (i = 0; i < ARRAY_SIZE(xpdd->evtchn_pending_pvt[pcpu]); i++) {
132 if (xpdd->evtchn_pending_pvt[pcpu][i]) {
133 FUNCTION_MSG("Unacknowledged event word = %d, val = %p\n", i, xpdd->evtchn_pending_pvt[pcpu][i]);
134 xpdd->evtchn_pending_pvt[pcpu][i] = 0;
135 }
136 }
138 vcpu_info = &shared_info_area->vcpu_info[vcpu];
140 vcpu_info->evtchn_upcall_pending = 0;
142 if (xpdd->interrupts_masked)
143 {
144 return TRUE;
145 }
147 evt_words = (xen_ulong_t)xchg((volatile xen_long_t *)&vcpu_info->evtchn_pending_sel, 0);
149 while (bit_scan_forward(&evt_word, evt_words))
150 {
151 evt_words &= ~(1 << evt_word);
152 while (bit_scan_forward(&evt_bit, shared_info_area->evtchn_pending[evt_word] & ~shared_info_area->evtchn_mask[evt_word]))
153 {
154 synch_clear_bit(evt_bit, (volatile xen_long_t *)&shared_info_area->evtchn_pending[evt_word]);
155 handled = TRUE;
156 port = (evt_word << BITS_PER_LONG_SHIFT) + evt_bit;
157 ev_action = &xpdd->ev_actions[port];
158 ev_action->count++;
159 switch (ev_action->type)
160 {
161 case EVT_ACTION_TYPE_NORMAL:
162 //FUNCTION_MSG("EVT_ACTION_TYPE_NORMAL port = %d\n", port);
163 ev_action->ServiceRoutine(ev_action->ServiceContext);
164 break;
165 case EVT_ACTION_TYPE_DPC:
166 //FUNCTION_MSG("EVT_ACTION_TYPE_DPC port = %d\n", port);
167 KeInsertQueueDpc(&ev_action->Dpc, NULL, NULL);
168 break;
169 #if 0
170 case EVT_ACTION_TYPE_SUSPEND:
171 FUNCTION_MSG("EVT_ACTION_TYPE_SUSPEND\n");
172 for (i = 0; i < NR_EVENTS; i++)
173 {
174 if (!(xpdd->ev_actions[i].flags & EVT_ACTION_FLAGS_NO_SUSPEND))
175 {
176 switch(xpdd->ev_actions[i].type)
177 {
178 #if 0
179 case EVT_ACTION_TYPE_IRQ:
180 {
181 int suspend_bit = i & (BITS_PER_LONG - 1);
182 int suspend_word = i >> BITS_PER_LONG_SHIFT;
183 synch_set_bit(suspend_bit, (volatile xen_long_t *)&xpdd->evtchn_pending_pvt[pcpu][suspend_word]);
184 }
185 break;
186 #endif
187 case EVT_ACTION_TYPE_NORMAL:
188 if (xpdd->ev_actions[i].ServiceRoutine)
189 {
190 xpdd->ev_actions[i].ServiceRoutine(xpdd->ev_actions[i].ServiceContext);
191 }
192 break;
193 case EVT_ACTION_TYPE_DPC:
194 KeInsertQueueDpc(&xpdd->ev_actions[i].Dpc, NULL, NULL);
195 break;
196 }
197 }
198 }
199 KeInsertQueueDpc(&ev_action->Dpc, NULL, NULL);
200 deferred = TRUE;
201 break;
202 #endif
203 default:
204 FUNCTION_MSG("Unhandled Event!!! port=%d\n", port);
205 break;
206 }
207 }
208 }
210 return handled && !deferred;
211 }
213 NTSTATUS
214 EvtChn_EvtInterruptEnable(WDFINTERRUPT interrupt, WDFDEVICE device)
215 {
216 NTSTATUS status = STATUS_SUCCESS;
218 UNREFERENCED_PARAMETER(interrupt);
219 UNREFERENCED_PARAMETER(device);
221 FUNCTION_ENTER();
222 FUNCTION_EXIT();
224 return status;
225 }
227 NTSTATUS
228 EvtChn_EvtInterruptDisable(WDFINTERRUPT interrupt, WDFDEVICE device)
229 {
230 NTSTATUS status = STATUS_SUCCESS;
232 UNREFERENCED_PARAMETER(interrupt);
233 UNREFERENCED_PARAMETER(device);
235 FUNCTION_ENTER();
236 FUNCTION_EXIT();
238 return status;
239 }
241 NTSTATUS
242 EvtChn_Bind(PVOID Context, evtchn_port_t port, PXN_EVENT_CALLBACK ServiceRoutine, PVOID ServiceContext, ULONG flags)
243 {
244 PXENPCI_DEVICE_DATA xpdd = Context;
245 ev_action_t *action = &xpdd->ev_actions[port];
247 FUNCTION_ENTER();
249 if (InterlockedCompareExchange((volatile LONG *)&action->type, EVT_ACTION_TYPE_NEW, EVT_ACTION_TYPE_EMPTY) != EVT_ACTION_TYPE_EMPTY) {
250 FUNCTION_MSG("Handler for port %d already registered\n", port);
251 return STATUS_UNSUCCESSFUL;
252 }
254 xpdd->ev_actions[port].ServiceRoutine = ServiceRoutine;
255 xpdd->ev_actions[port].ServiceContext = ServiceContext;
256 xpdd->ev_actions[port].xpdd = xpdd;
257 xpdd->ev_actions[port].flags = flags;
258 KeMemoryBarrier();
259 xpdd->ev_actions[port].type = EVT_ACTION_TYPE_NORMAL;
261 EvtChn_Unmask(Context, port);
263 FUNCTION_EXIT();
265 return STATUS_SUCCESS;
266 }
268 NTSTATUS
269 EvtChn_BindDpc(PVOID Context, evtchn_port_t port, PXN_EVENT_CALLBACK ServiceRoutine, PVOID ServiceContext, ULONG flags) {
270 PXENPCI_DEVICE_DATA xpdd = Context;
271 ev_action_t *action = &xpdd->ev_actions[port];
273 FUNCTION_ENTER();
275 if (InterlockedCompareExchange((volatile LONG *)&action->type, EVT_ACTION_TYPE_NEW, EVT_ACTION_TYPE_EMPTY) != EVT_ACTION_TYPE_EMPTY) {
276 FUNCTION_MSG(" Handler for port %d already registered\n", port);
277 return STATUS_UNSUCCESSFUL;
278 }
280 xpdd->ev_actions[port].ServiceRoutine = ServiceRoutine;
281 xpdd->ev_actions[port].ServiceContext = ServiceContext;
282 xpdd->ev_actions[port].xpdd = xpdd;
283 xpdd->ev_actions[port].flags = flags;
284 KeMemoryBarrier(); // make sure that the new service routine is only called once the context is set up
285 InterlockedExchange((volatile LONG *)&action->type, EVT_ACTION_TYPE_DPC);
287 EvtChn_Unmask(Context, port);
289 FUNCTION_EXIT();
291 return STATUS_SUCCESS;
292 }
294 #if 0
295 NTSTATUS
296 EvtChn_BindIrq(PVOID Context, evtchn_port_t port, ULONG vector, PCHAR description, ULONG flags)
297 {
298 PXENPCI_DEVICE_DATA xpdd = Context;
299 ev_action_t *action = &xpdd->ev_actions[port];
301 FUNCTION_ENTER();
303 if (InterlockedCompareExchange((volatile LONG *)&action->type, EVT_ACTION_TYPE_NEW, EVT_ACTION_TYPE_EMPTY) != EVT_ACTION_TYPE_EMPTY)
304 {
305 FUNCTION_MSG("Handler for port %d already registered\n", port);
306 return STATUS_UNSUCCESSFUL;
307 }
309 xpdd->ev_actions[port].vector = vector;
310 xpdd->ev_actions[port].xpdd = xpdd;
311 KeMemoryBarrier();
312 xpdd->ev_actions[port].type = EVT_ACTION_TYPE_IRQ;
313 RtlStringCbCopyA(xpdd->ev_actions[port].description, 128, description);
314 xpdd->ev_actions[port].flags = flags;
316 EvtChn_Unmask(Context, port);
318 FUNCTION_EXIT();
320 return STATUS_SUCCESS;
321 }
322 #endif
324 NTSTATUS
325 EvtChn_Unbind(PVOID Context, evtchn_port_t port) {
326 PXENPCI_DEVICE_DATA xpdd = Context;
327 ev_action_t *action = &xpdd->ev_actions[port];
328 int old_type;
330 EvtChn_Mask(Context, port);
331 old_type = InterlockedExchange((volatile LONG *)&action->type, EVT_ACTION_TYPE_EMPTY);
333 if (old_type == EVT_ACTION_TYPE_DPC) { // || old_type == EVT_ACTION_TYPE_SUSPEND) {
334 KeRemoveQueueDpc(&xpdd->ev_actions[port].Dpc);
335 #if (NTDDI_VERSION >= NTDDI_WINXP)
336 KeFlushQueuedDpcs();
337 #endif
338 }
340 KeMemoryBarrier(); // make sure we don't call the old Service Routine with the new data...
341 xpdd->ev_actions[port].ServiceRoutine = NULL;
342 xpdd->ev_actions[port].ServiceContext = NULL;
344 return STATUS_SUCCESS;
345 }
347 NTSTATUS
348 EvtChn_Mask(PVOID Context, evtchn_port_t port) {
349 PXENPCI_DEVICE_DATA xpdd = Context;
351 synch_set_bit(port & (BITS_PER_LONG - 1),
352 (volatile xen_long_t *)&xpdd->shared_info_area->evtchn_mask[port >> BITS_PER_LONG_SHIFT]);
353 return STATUS_SUCCESS;
354 }
356 NTSTATUS
357 EvtChn_Unmask(PVOID context, evtchn_port_t port) {
358 PXENPCI_DEVICE_DATA xpdd = context;
360 synch_clear_bit(port & (BITS_PER_LONG - 1),
361 (volatile xen_long_t *)&xpdd->shared_info_area->evtchn_mask[port >> BITS_PER_LONG_SHIFT]);
362 return STATUS_SUCCESS;
363 }
365 NTSTATUS
366 EvtChn_Notify(PVOID context, evtchn_port_t port) {
367 struct evtchn_send send;
369 UNREFERENCED_PARAMETER(context);
370 send.port = port;
371 (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
372 return STATUS_SUCCESS;
373 }
375 evtchn_port_t
376 EvtChn_AllocIpi(PVOID context, ULONG vcpu) {
377 evtchn_bind_ipi_t op;
379 UNREFERENCED_PARAMETER(context);
380 FUNCTION_ENTER();
381 op.vcpu = vcpu;
382 op.port = 0;
383 HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &op);
384 FUNCTION_EXIT();
385 return op.port;
386 }
388 evtchn_port_t
389 EvtChn_AllocUnbound(PVOID context, domid_t Domain) {
390 evtchn_alloc_unbound_t op;
392 UNREFERENCED_PARAMETER(context);
393 op.dom = DOMID_SELF;
394 op.remote_dom = Domain;
395 HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, &op);
396 return op.port;
397 }
399 VOID
400 EvtChn_Close(PVOID context, evtchn_port_t port) {
401 evtchn_close_t op;
403 UNREFERENCED_PARAMETER(context);
404 op.port = port;
405 HYPERVISOR_event_channel_op(EVTCHNOP_close, &op);
406 return;
407 }
409 #if 0
410 VOID
411 EvtChn_PdoEventChannelDpc(PVOID context)
412 {
413 PXENPCI_DEVICE_DATA xpdd = context;
415 FUNCTION_ENTER();
416 KeSetEvent(&xpdd->pdo_suspend_event, IO_NO_INCREMENT, FALSE);
417 FUNCTION_EXIT();
418 }
419 #endif
421 NTSTATUS
422 EvtChn_Init(PXENPCI_DEVICE_DATA xpdd)
423 {
424 ULONGLONG result;
425 ev_action_t *action;
426 int i;
428 FUNCTION_ENTER();
430 for (i = 0; i < NR_EVENTS; i++)
431 {
432 EvtChn_Mask(xpdd, i);
433 action = &xpdd->ev_actions[i];
434 action->type = EVT_ACTION_TYPE_EMPTY;
435 //action->port = -1;
436 action->count = 0;
437 KeInitializeDpc(&action->Dpc, EvtChn_DpcBounce, action);
438 }
440 for (i = 0; i < 8; i++) {
441 xpdd->shared_info_area->evtchn_pending[i] = 0;
442 }
444 for (i = 0; i < MAX_VIRT_CPUS; i++) {
445 xpdd->shared_info_area->vcpu_info[i].evtchn_upcall_pending = 0;
446 xpdd->shared_info_area->vcpu_info[i].evtchn_pending_sel = 0;
447 xpdd->shared_info_area->vcpu_info[i].evtchn_upcall_mask = 1; /* apparantly this doesn't do anything */
448 }
450 KeMemoryBarrier();
452 result = hvm_set_parameter(HVM_PARAM_CALLBACK_IRQ, xpdd->irq_number);
453 FUNCTION_MSG("hvm_set_parameter(HVM_PARAM_CALLBACK_IRQ, %d) = %d\n", xpdd->irq_number, (ULONG)result);
455 for (i = 0; i < MAX_VIRT_CPUS; i++)
456 xpdd->shared_info_area->vcpu_info[i].evtchn_upcall_mask = 0;
457 xpdd->interrupts_masked = FALSE;
458 KeMemoryBarrier();
460 #if 0
461 KeInitializeEvent(&xpdd->pdo_suspend_event, SynchronizationEvent, FALSE);
462 xpdd->pdo_event_channel = EvtChn_AllocIpi(xpdd, 0);
463 EvtChn_BindDpc(xpdd, xpdd->pdo_event_channel, EvtChn_PdoEventChannelDpc, xpdd, EVT_ACTION_FLAGS_DEFAULT);
464 xpdd->ev_actions[xpdd->pdo_event_channel].type = EVT_ACTION_TYPE_SUSPEND; /* override dpc type */
465 FUNCTION_MSG("pdo_event_channel = %d\n", xpdd->pdo_event_channel);
466 #endif
468 FUNCTION_EXIT();
470 return STATUS_SUCCESS;
471 }
473 NTSTATUS
474 EvtChn_Suspend(PXENPCI_DEVICE_DATA xpdd)
475 {
476 int i;
478 xpdd->interrupts_masked = TRUE;
479 for (i = 0; i < MAX_VIRT_CPUS; i++)
480 xpdd->shared_info_area->vcpu_info[i].evtchn_upcall_mask = 1;
481 KeMemoryBarrier();
482 hvm_set_parameter(HVM_PARAM_CALLBACK_IRQ, 0);
484 for (i = 0; i < NR_EVENTS; i++) {
485 if (xpdd->ev_actions[i].type == EVT_ACTION_TYPE_DPC) {
486 KeRemoveQueueDpc(&xpdd->ev_actions[i].Dpc);
487 }
488 }
489 #if (NTDDI_VERSION >= NTDDI_WINXP)
490 KeFlushQueuedDpcs();
491 #endif
492 return STATUS_SUCCESS;
493 }
495 NTSTATUS
496 EvtChn_Resume(PXENPCI_DEVICE_DATA xpdd)
497 {
498 return EvtChn_Init(xpdd);