From bdd7e812d40ea9420e0056f25009cee2e350e0de Mon Sep 17 00:00:00 2001 From: Julien Grall Date: Tue, 14 Jan 2014 01:41:04 +0000 Subject: [PATCH 27/48] xen: move xen_intr.c to common code --- sys/conf/files | 1 + sys/conf/files.i386 | 1 - sys/x86/xen/xen_intr.c | 1626 ------------------------------------------------ sys/xen/xen_intr.c | 1626 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1627 insertions(+), 1627 deletions(-) delete mode 100644 sys/x86/xen/xen_intr.c create mode 100644 sys/xen/xen_intr.c diff --git a/sys/conf/files b/sys/conf/files index 730a3c8..d9ba79e 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -3994,6 +3994,7 @@ xen/xenbus/xenbusb_if.m optional xen | xenhvm xen/xenbus/xenbusb.c optional xen | xenhvm xen/xenbus/xenbusb_front.c optional xen | xenhvm xen/xenbus/xenbusb_back.c optional xen | xenhvm +xen/xen_intr.c optional xen | xenhvm xdr/xdr.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd xdr/xdr_array.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd xdr/xdr_mbuf.c optional krpc | nfslockd | nfsclient | nfsserver | nfscl | nfsd diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index 93d3895..1dea308 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -596,7 +596,6 @@ x86/x86/nexus.c standard x86/x86/tsc.c standard x86/x86/delay.c standard x86/xen/hvm.c optional xenhvm -x86/xen/xen_intr.c optional xen | xenhvm x86/xen/xen_apic.c optional xenhvm x86/xen/xen_nexus.c optional xen | xenhvm x86/xen/xen_msi.c optional xen | xenhvm diff --git a/sys/x86/xen/xen_intr.c b/sys/x86/xen/xen_intr.c deleted file mode 100644 index 3bd292e..0000000 --- a/sys/x86/xen/xen_intr.c +++ /dev/null @@ -1,1626 +0,0 @@ -/****************************************************************************** - * xen_intr.c - * - * Xen event and interrupt services for x86 PV and HVM guests. - * - * Copyright (c) 2002-2005, K A Fraser - * Copyright (c) 2005, Intel Corporation - * Copyright (c) 2012, Spectra Logic Corporation - * - * This file may be distributed separately from the Linux kernel, or - * incorporated into other software packages, subject to the following license: - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this source file (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, modify, - * merge, publish, distribute, sublicense, and/or sell copies of the Software, - * and to permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include "opt_ddb.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include -#include - -#ifdef DDB -#include -#endif - -static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); - -/** - * Per-cpu event channel processing state. - */ -struct xen_intr_pcpu_data { - /** - * The last event channel bitmap section (level one bit) processed. - * This is used to ensure we scan all ports before - * servicing an already servied port again. - */ - u_int last_processed_l1i; - - /** - * The last event channel processed within the event channel - * bitmap being scanned. - */ - u_int last_processed_l2i; - - /** Pointer to this CPU's interrupt statistic counter. */ - u_long *evtchn_intrcnt; - - /** - * A bitmap of ports that can be serviced from this CPU. - * A set bit means interrupt handling is enabled. - */ - u_long evtchn_enabled[sizeof(u_long) * 8]; -}; - -/* - * Start the scan at port 0 by initializing the last scanned - * location as the highest numbered event channel port. - */ -DPCPU_DEFINE(struct xen_intr_pcpu_data, xen_intr_pcpu) = { - .last_processed_l1i = LONG_BIT - 1, - .last_processed_l2i = LONG_BIT - 1 -}; - -DPCPU_DECLARE(struct vcpu_info *, vcpu_info); - -#define XEN_EEXIST 17 /* Xen "already exists" error */ -#define XEN_ALLOCATE_VECTOR 0 /* Allocate a vector for this event channel */ -#define XEN_INVALID_EVTCHN 0 /* Invalid event channel */ - -#define is_valid_evtchn(x) ((x) != XEN_INVALID_EVTCHN) - -struct xenisrc { - struct intsrc xi_intsrc; - enum evtchn_type xi_type; - int xi_cpu; /* VCPU for delivery. */ - int xi_vector; /* Global isrc vector number. */ - evtchn_port_t xi_port; - int xi_pirq; - int xi_virq; - void *xi_cookie; - u_int xi_close:1; /* close on unbind? */ - u_int xi_shared:1; /* Shared with other domains. */ - u_int xi_activehi:1; - u_int xi_edgetrigger:1; -}; - -#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) - -static void xen_intr_suspend(struct pic *); -static void xen_intr_resume(struct pic *, bool suspend_cancelled); -static void xen_intr_enable_source(struct intsrc *isrc); -static void xen_intr_disable_source(struct intsrc *isrc, int eoi); -static void xen_intr_eoi_source(struct intsrc *isrc); -static void xen_intr_enable_intr(struct intsrc *isrc); -static void xen_intr_disable_intr(struct intsrc *isrc); -static int xen_intr_vector(struct intsrc *isrc); -static int xen_intr_source_pending(struct intsrc *isrc); -static int xen_intr_config_intr(struct intsrc *isrc, - enum intr_trigger trig, enum intr_polarity pol); -static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id); - -static void xen_intr_pirq_enable_source(struct intsrc *isrc); -static void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi); -static void xen_intr_pirq_eoi_source(struct intsrc *isrc); -static void xen_intr_pirq_enable_intr(struct intsrc *isrc); -static void xen_intr_pirq_disable_intr(struct intsrc *isrc); -static int xen_intr_pirq_config_intr(struct intsrc *isrc, - enum intr_trigger trig, enum intr_polarity pol); - -/** - * PIC interface for all event channel port types except physical IRQs. - */ -struct pic xen_intr_pic = { - .pic_enable_source = xen_intr_enable_source, - .pic_disable_source = xen_intr_disable_source, - .pic_eoi_source = xen_intr_eoi_source, - .pic_enable_intr = xen_intr_enable_intr, - .pic_disable_intr = xen_intr_disable_intr, - .pic_vector = xen_intr_vector, - .pic_source_pending = xen_intr_source_pending, - .pic_suspend = xen_intr_suspend, - .pic_resume = xen_intr_resume, - .pic_config_intr = xen_intr_config_intr, - .pic_assign_cpu = xen_intr_assign_cpu -}; - -/** - * PIC interface for all event channel representing - * physical interrupt sources. - */ -struct pic xen_intr_pirq_pic = { - .pic_enable_source = xen_intr_pirq_enable_source, - .pic_disable_source = xen_intr_pirq_disable_source, - .pic_eoi_source = xen_intr_pirq_eoi_source, - .pic_enable_intr = xen_intr_pirq_enable_intr, - .pic_disable_intr = xen_intr_pirq_disable_intr, - .pic_vector = xen_intr_vector, - .pic_source_pending = xen_intr_source_pending, - .pic_config_intr = xen_intr_pirq_config_intr, - .pic_assign_cpu = xen_intr_assign_cpu -}; - -static struct mtx xen_intr_isrc_lock; -static int xen_intr_auto_vector_count; -static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS]; -static u_long *xen_intr_pirq_eoi_map; -static boolean_t xen_intr_pirq_eoi_map_enabled; - -/*------------------------- Private Functions --------------------------------*/ -/** - * Disable signal delivery for an event channel port on the - * specified CPU. - * - * \param port The event channel port to mask. - * - * This API is used to manage the port<=>CPU binding of event - * channel handlers. - * - * \note This operation does not preclude reception of an event - * for this event channel on another CPU. To mask the - * event channel globally, use evtchn_mask(). - */ -static inline void -evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port) -{ - struct xen_intr_pcpu_data *pcpu; - - pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); - clear_bit(port, pcpu->evtchn_enabled); -} - -/** - * Enable signal delivery for an event channel port on the - * specified CPU. - * - * \param port The event channel port to unmask. - * - * This API is used to manage the port<=>CPU binding of event - * channel handlers. - * - * \note This operation does not guarantee that event delivery - * is enabled for this event channel port. The port must - * also be globally enabled. See evtchn_unmask(). - */ -static inline void -evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port) -{ - struct xen_intr_pcpu_data *pcpu; - - pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); - set_bit(port, pcpu->evtchn_enabled); -} - -/** - * Allocate and register a per-cpu Xen upcall interrupt counter. - * - * \param cpu The cpu for which to register this interrupt count. - */ -static void -xen_intr_intrcnt_add(u_int cpu) -{ - char buf[MAXCOMLEN + 1]; - struct xen_intr_pcpu_data *pcpu; - - pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); - if (pcpu->evtchn_intrcnt != NULL) - return; - - snprintf(buf, sizeof(buf), "cpu%d:xen", cpu); - intrcnt_add(buf, &pcpu->evtchn_intrcnt); -} - -/** - * Search for an already allocated but currently unused Xen interrupt - * source object. - * - * \param type Restrict the search to interrupt sources of the given - * type. - * - * \return A pointer to a free Xen interrupt source object or NULL. - */ -static struct xenisrc * -xen_intr_find_unused_isrc(enum evtchn_type type) -{ - int isrc_idx; - - KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held")); - - for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) { - struct xenisrc *isrc; - u_int vector; - - vector = FIRST_EVTCHN_INT + isrc_idx; - isrc = (struct xenisrc *)intr_lookup_source(vector); - if (isrc != NULL - && isrc->xi_type == EVTCHN_TYPE_UNBOUND) { - KASSERT(isrc->xi_intsrc.is_handlers == 0, - ("Free evtchn still has handlers")); - isrc->xi_type = type; - return (isrc); - } - } - return (NULL); -} - -/** - * Allocate a Xen interrupt source object. - * - * \param type The type of interrupt source to create. - * - * \return A pointer to a newly allocated Xen interrupt source - * object or NULL. - */ -static struct xenisrc * -xen_intr_alloc_isrc(enum evtchn_type type, int vector) -{ - static int warned; - struct xenisrc *isrc; - - KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held")); - - if (xen_intr_auto_vector_count > NR_EVENT_CHANNELS) { - if (!warned) { - warned = 1; - printf("xen_intr_alloc: Event channels exhausted.\n"); - } - return (NULL); - } - - if (type != EVTCHN_TYPE_PIRQ) { - vector = FIRST_EVTCHN_INT + xen_intr_auto_vector_count; - xen_intr_auto_vector_count++; - } - - KASSERT((intr_lookup_source(vector) == NULL), - ("Trying to use an already allocated vector")); - - mtx_unlock(&xen_intr_isrc_lock); - isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO); - isrc->xi_intsrc.is_pic = - (type == EVTCHN_TYPE_PIRQ) ? &xen_intr_pirq_pic : &xen_intr_pic; - isrc->xi_vector = vector; - isrc->xi_type = type; - intr_register_source(&isrc->xi_intsrc); - mtx_lock(&xen_intr_isrc_lock); - - return (isrc); -} - -/** - * Attempt to free an active Xen interrupt source object. - * - * \param isrc The interrupt source object to release. - * - * \returns EBUSY if the source is still in use, otherwise 0. - */ -static int -xen_intr_release_isrc(struct xenisrc *isrc) -{ - - mtx_lock(&xen_intr_isrc_lock); - if (isrc->xi_intsrc.is_handlers != 0) { - mtx_unlock(&xen_intr_isrc_lock); - return (EBUSY); - } - evtchn_mask_port(isrc->xi_port); - evtchn_clear_port(isrc->xi_port); - - /* Rebind port to CPU 0. */ - evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); - evtchn_cpu_unmask_port(0, isrc->xi_port); - - if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) { - struct evtchn_close close = { .port = isrc->xi_port }; - if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) - panic("EVTCHNOP_close failed"); - } - - xen_intr_port_to_isrc[isrc->xi_port] = NULL; - isrc->xi_cpu = 0; - isrc->xi_type = EVTCHN_TYPE_UNBOUND; - isrc->xi_port = 0; - isrc->xi_cookie = NULL; - mtx_unlock(&xen_intr_isrc_lock); - return (0); -} - -/** - * Associate an interrupt handler with an already allocated local Xen - * event channel port. - * - * \param isrcp The returned Xen interrupt object associated with - * the specified local port. - * \param local_port The event channel to bind. - * \param type The event channel type of local_port. - * \param intr_owner The device making this bind request. - * \param filter An interrupt filter handler. Specify NULL - * to always dispatch to the ithread handler. - * \param handler An interrupt ithread handler. Optional (can - * specify NULL) if all necessary event actions - * are performed by filter. - * \param arg Argument to present to both filter and handler. - * \param irqflags Interrupt handler flags. See sys/bus.h. - * \param handlep Pointer to an opaque handle used to manage this - * registration. - * - * \returns 0 on success, otherwise an errno. - */ -static int -xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port, - enum evtchn_type type, device_t intr_owner, driver_filter_t filter, - driver_intr_t handler, void *arg, enum intr_type flags, - xen_intr_handle_t *port_handlep) -{ - struct xenisrc *isrc; - int error; - - *isrcp = NULL; - if (port_handlep == NULL) { - device_printf(intr_owner, - "xen_intr_bind_isrc: Bad event handle\n"); - return (EINVAL); - } - - mtx_lock(&xen_intr_isrc_lock); - isrc = xen_intr_find_unused_isrc(type); - if (isrc == NULL) { - isrc = xen_intr_alloc_isrc(type, XEN_ALLOCATE_VECTOR); - if (isrc == NULL) { - mtx_unlock(&xen_intr_isrc_lock); - return (ENOSPC); - } - } - isrc->xi_port = local_port; - xen_intr_port_to_isrc[local_port] = isrc; - mtx_unlock(&xen_intr_isrc_lock); - - /* Assign the opaque handler (the event channel port) */ - *port_handlep = &isrc->xi_port; - - if (filter == NULL && handler == NULL) { - /* - * No filter/handler provided, leave the event channel - * masked and without a valid handler, the caller is - * in charge of setting that up. - */ - *isrcp = isrc; - return (0); - } - - error = xen_intr_add_handler(intr_owner, filter, handler, arg, flags, - *port_handlep); - if (error != 0) { - xen_intr_release_isrc(isrc); - return (error); - } - *isrcp = isrc; - return (0); -} - -/** - * Lookup a Xen interrupt source object given an interrupt binding handle. - * - * \param handle A handle initialized by a previous call to - * xen_intr_bind_isrc(). - * - * \returns A pointer to the Xen interrupt source object associated - * with the given interrupt handle. NULL if no association - * currently exists. - */ -static struct xenisrc * -xen_intr_isrc(xen_intr_handle_t handle) -{ - evtchn_port_t port; - - if (handle == NULL) - return (NULL); - - port = *(evtchn_port_t *)handle; - if (!is_valid_evtchn(port) || port >= NR_EVENT_CHANNELS) - return (NULL); - - return (xen_intr_port_to_isrc[port]); -} - -/** - * Determine the event channel ports at the given section of the - * event port bitmap which have pending events for the given cpu. - * - * \param pcpu The Xen interrupt pcpu data for the cpu being querried. - * \param sh The Xen shared info area. - * \param idx The index of the section of the event channel bitmap to - * inspect. - * - * \returns A u_long with bits set for every event channel with pending - * events. - */ -static inline u_long -xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh, - u_int idx) -{ - return (sh->evtchn_pending[idx] - & ~sh->evtchn_mask[idx] - & pcpu->evtchn_enabled[idx]); -} - -/** - * Interrupt handler for processing all Xen event channel events. - * - * \param trap_frame The trap frame context for the current interrupt. - */ -void -xen_intr_handle_upcall(struct trapframe *trap_frame) -{ - u_int l1i, l2i, port, cpu; - u_long masked_l1, masked_l2; - struct xenisrc *isrc; - shared_info_t *s; - vcpu_info_t *v; - struct xen_intr_pcpu_data *pc; - u_long l1, l2; - - /* - * Disable preemption in order to always check and fire events - * on the right vCPU - */ - critical_enter(); - - cpu = PCPU_GET(cpuid); - pc = DPCPU_PTR(xen_intr_pcpu); - s = HYPERVISOR_shared_info; - v = DPCPU_GET(vcpu_info); - - if (xen_hvm_domain() && !xen_vector_callback_enabled) { - KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU")); - } - - v->evtchn_upcall_pending = 0; - -#if 0 -#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */ - /* Clear master flag /before/ clearing selector flag. */ - wmb(); -#endif -#endif - - l1 = atomic_readandclear_long(&v->evtchn_pending_sel); - - l1i = pc->last_processed_l1i; - l2i = pc->last_processed_l2i; - (*pc->evtchn_intrcnt)++; - - while (l1 != 0) { - - l1i = (l1i + 1) % LONG_BIT; - masked_l1 = l1 & ((~0UL) << l1i); - - if (masked_l1 == 0) { - /* - * if we masked out all events, wrap around - * to the beginning. - */ - l1i = LONG_BIT - 1; - l2i = LONG_BIT - 1; - continue; - } - l1i = ffsl(masked_l1) - 1; - - do { - l2 = xen_intr_active_ports(pc, s, l1i); - - l2i = (l2i + 1) % LONG_BIT; - masked_l2 = l2 & ((~0UL) << l2i); - - if (masked_l2 == 0) { - /* if we masked out all events, move on */ - l2i = LONG_BIT - 1; - break; - } - l2i = ffsl(masked_l2) - 1; - - /* process port */ - port = (l1i * LONG_BIT) + l2i; - synch_clear_bit(port, &s->evtchn_pending[0]); - - isrc = xen_intr_port_to_isrc[port]; - if (__predict_false(isrc == NULL)) - continue; - - /* Make sure we are firing on the right vCPU */ - KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), - ("Received unexpected event on vCPU#%d, event bound to vCPU#%d", - PCPU_GET(cpuid), isrc->xi_cpu)); - - intr_execute_handlers(&isrc->xi_intsrc, trap_frame); - - /* - * If this is the final port processed, - * we'll pick up here+1 next time. - */ - pc->last_processed_l1i = l1i; - pc->last_processed_l2i = l2i; - - } while (l2i != LONG_BIT - 1); - - l2 = xen_intr_active_ports(pc, s, l1i); - if (l2 == 0) { - /* - * We handled all ports, so we can clear the - * selector bit. - */ - l1 &= ~(1UL << l1i); - } - } - critical_exit(); -} - -static int -xen_intr_init(void *dummy __unused) -{ - shared_info_t *s = HYPERVISOR_shared_info; - struct xen_intr_pcpu_data *pcpu; - struct physdev_pirq_eoi_gmfn eoi_gmfn; - int i, rc; - - if (!xen_domain()) - return (0); - - mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); - - /* - * Register interrupt count manually as we aren't - * guaranteed to see a call to xen_intr_assign_cpu() - * before our first interrupt. Also set the per-cpu - * mask of CPU#0 to enable all, since by default - * all event channels are bound to CPU#0. - */ - CPU_FOREACH(i) { - pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); - memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, - sizeof(pcpu->evtchn_enabled)); - xen_intr_intrcnt_add(i); - } - - for (i = 0; i < nitems(s->evtchn_mask); i++) - atomic_store_rel_long(&s->evtchn_mask[i], ~0); - - /* Try to register PIRQ EOI map */ - xen_intr_pirq_eoi_map = malloc(PAGE_SIZE, M_XENINTR, M_WAITOK | M_ZERO); - eoi_gmfn.gmfn = atop(vtophys(xen_intr_pirq_eoi_map)); - rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn); - if (rc != 0 && bootverbose) - printf("Xen interrupts: unable to register PIRQ EOI map\n"); - else - xen_intr_pirq_eoi_map_enabled = true; - - intr_register_pic(&xen_intr_pic); - intr_register_pic(&xen_intr_pirq_pic); - - if (bootverbose) - printf("Xen interrupt system initialized\n"); - - return (0); -} -SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL); - -/*--------------------------- Common PIC Functions ---------------------------*/ -/** - * Prepare this PIC for system suspension. - */ -static void -xen_intr_suspend(struct pic *unused) -{ -} - -static void -xen_rebind_ipi(struct xenisrc *isrc) -{ -#ifdef SMP - int cpu = isrc->xi_cpu; - int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; - int error; - struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; - - error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, - &bind_ipi); - if (error != 0) - panic("unable to rebind xen IPI: %d", error); - - isrc->xi_port = bind_ipi.port; - isrc->xi_cpu = 0; - xen_intr_port_to_isrc[bind_ipi.port] = isrc; - - error = xen_intr_assign_cpu(&isrc->xi_intsrc, - cpu_apic_ids[cpu]); - if (error) - panic("unable to bind xen IPI to CPU#%d: %d", - cpu, error); - - evtchn_unmask_port(bind_ipi.port); -#else - panic("Resume IPI event channel on UP"); -#endif -} - -static void -xen_rebind_virq(struct xenisrc *isrc) -{ - int cpu = isrc->xi_cpu; - int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; - int error; - struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq, - .vcpu = vcpu_id }; - - error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, - &bind_virq); - if (error != 0) - panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error); - - isrc->xi_port = bind_virq.port; - isrc->xi_cpu = 0; - xen_intr_port_to_isrc[bind_virq.port] = isrc; - -#ifdef SMP - error = xen_intr_assign_cpu(&isrc->xi_intsrc, - cpu_apic_ids[cpu]); - if (error) - panic("unable to bind xen VIRQ#%d to CPU#%d: %d", - isrc->xi_virq, cpu, error); -#endif - - evtchn_unmask_port(bind_virq.port); -} - -/** - * Return this PIC to service after being suspended. - */ -static void -xen_intr_resume(struct pic *unused, bool suspend_cancelled) -{ - shared_info_t *s = HYPERVISOR_shared_info; - struct xenisrc *isrc; - u_int isrc_idx; - int i; - - if (suspend_cancelled) - return; - - /* Reset the per-CPU masks */ - CPU_FOREACH(i) { - struct xen_intr_pcpu_data *pcpu; - - pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); - memset(pcpu->evtchn_enabled, - i == 0 ? ~0 : 0, sizeof(pcpu->evtchn_enabled)); - } - - /* Mask all event channels. */ - for (i = 0; i < nitems(s->evtchn_mask); i++) - atomic_store_rel_long(&s->evtchn_mask[i], ~0); - - /* Remove port -> isrc mappings */ - memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc)); - - /* Free unused isrcs and rebind VIRQs and IPIs */ - for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx++) { - u_int vector; - - vector = FIRST_EVTCHN_INT + isrc_idx; - isrc = (struct xenisrc *)intr_lookup_source(vector); - if (isrc != NULL) { - isrc->xi_port = 0; - switch (isrc->xi_type) { - case EVTCHN_TYPE_IPI: - xen_rebind_ipi(isrc); - break; - case EVTCHN_TYPE_VIRQ: - xen_rebind_virq(isrc); - break; - default: - isrc->xi_cpu = 0; - break; - } - } - } -} - -/** - * Disable a Xen interrupt source. - * - * \param isrc The interrupt source to disable. - */ -static void -xen_intr_disable_intr(struct intsrc *base_isrc) -{ - struct xenisrc *isrc = (struct xenisrc *)base_isrc; - - evtchn_mask_port(isrc->xi_port); -} - -/** - * Determine the global interrupt vector number for - * a Xen interrupt source. - * - * \param isrc The interrupt source to query. - * - * \return The vector number corresponding to the given interrupt source. - */ -static int -xen_intr_vector(struct intsrc *base_isrc) -{ - struct xenisrc *isrc = (struct xenisrc *)base_isrc; - - return (isrc->xi_vector); -} - -/** - * Determine whether or not interrupt events are pending on the - * the given interrupt source. - * - * \param isrc The interrupt source to query. - * - * \returns 0 if no events are pending, otherwise non-zero. - */ -static int -xen_intr_source_pending(struct intsrc *isrc) -{ - /* - * EventChannels are edge triggered and never masked. - * There can be no pending events. - */ - return (0); -} - -/** - * Perform configuration of an interrupt source. - * - * \param isrc The interrupt source to configure. - * \param trig Edge or level. - * \param pol Active high or low. - * - * \returns 0 if no events are pending, otherwise non-zero. - */ -static int -xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig, - enum intr_polarity pol) -{ - /* Configuration is only possible via the evtchn apis. */ - return (ENODEV); -} - -/** - * Configure CPU affinity for interrupt source event delivery. - * - * \param isrc The interrupt source to configure. - * \param apic_id The apic id of the CPU for handling future events. - * - * \returns 0 if successful, otherwise an errno. - */ -static int -xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id) -{ -#ifdef SMP - struct evtchn_bind_vcpu bind_vcpu; - struct xenisrc *isrc; - u_int to_cpu, vcpu_id; - int error, masked; - -#ifdef XENHVM - if (xen_vector_callback_enabled == 0) - return (EOPNOTSUPP); -#endif - - to_cpu = apic_cpuid(apic_id); - vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id; - xen_intr_intrcnt_add(to_cpu); - - mtx_lock(&xen_intr_isrc_lock); - isrc = (struct xenisrc *)base_isrc; - if (!is_valid_evtchn(isrc->xi_port)) { - mtx_unlock(&xen_intr_isrc_lock); - return (EINVAL); - } - - /* - * Mask the event channel while binding it to prevent interrupt - * delivery with an inconsistent state in isrc->xi_cpu. - */ - masked = evtchn_test_and_set_mask(isrc->xi_port); - if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) || - (isrc->xi_type == EVTCHN_TYPE_IPI)) { - /* - * Virtual IRQs are associated with a cpu by - * the Hypervisor at evtchn_bind_virq time, so - * all we need to do is update the per-CPU masks. - */ - evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); - isrc->xi_cpu = to_cpu; - evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); - goto out; - } - - bind_vcpu.port = isrc->xi_port; - bind_vcpu.vcpu = vcpu_id; - - error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu); - if (isrc->xi_cpu != to_cpu) { - if (error == 0) { - /* Commit to new binding by removing the old one. */ - evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); - isrc->xi_cpu = to_cpu; - evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); - } - } - -out: - if (masked == 0) - evtchn_unmask_port(isrc->xi_port); - mtx_unlock(&xen_intr_isrc_lock); - return (0); -#else - return (EOPNOTSUPP); -#endif -} - -/*------------------- Virtual Interrupt Source PIC Functions -----------------*/ -/* - * Mask a level triggered interrupt source. - * - * \param isrc The interrupt source to mask (if necessary). - * \param eoi If non-zero, perform any necessary end-of-interrupt - * acknowledgements. - */ -static void -xen_intr_disable_source(struct intsrc *isrc, int eoi) -{ -} - -/* - * Unmask a level triggered interrupt source. - * - * \param isrc The interrupt source to unmask (if necessary). - */ -static void -xen_intr_enable_source(struct intsrc *isrc) -{ -} - -/* - * Perform any necessary end-of-interrupt acknowledgements. - * - * \param isrc The interrupt source to EOI. - */ -static void -xen_intr_eoi_source(struct intsrc *isrc) -{ -} - -/* - * Enable and unmask the interrupt source. - * - * \param isrc The interrupt source to enable. - */ -static void -xen_intr_enable_intr(struct intsrc *base_isrc) -{ - struct xenisrc *isrc = (struct xenisrc *)base_isrc; - - evtchn_unmask_port(isrc->xi_port); -} - -/*------------------ Physical Interrupt Source PIC Functions -----------------*/ -/* - * Mask a level triggered interrupt source. - * - * \param isrc The interrupt source to mask (if necessary). - * \param eoi If non-zero, perform any necessary end-of-interrupt - * acknowledgements. - */ -static void -xen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi) -{ - struct xenisrc *isrc; - - isrc = (struct xenisrc *)base_isrc; - evtchn_mask_port(isrc->xi_port); - - if (eoi == PIC_EOI) - xen_intr_pirq_eoi_source(base_isrc); -} - -/* - * Unmask a level triggered interrupt source. - * - * \param isrc The interrupt source to unmask (if necessary). - */ -static void -xen_intr_pirq_enable_source(struct intsrc *base_isrc) -{ - struct xenisrc *isrc; - - isrc = (struct xenisrc *)base_isrc; - evtchn_unmask_port(isrc->xi_port); -} - -/* - * Perform any necessary end-of-interrupt acknowledgements. - * - * \param isrc The interrupt source to EOI. - */ -static void -xen_intr_pirq_eoi_source(struct intsrc *base_isrc) -{ - struct xenisrc *isrc; - - /* XXX Use shared page of flags for this. */ - isrc = (struct xenisrc *)base_isrc; - if (test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map)) { - struct physdev_eoi eoi = { .irq = isrc->xi_pirq }; - - (void)HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi); - } -} - -/* - * Enable and unmask the interrupt source. - * - * \param isrc The interrupt source to enable. - */ -static void -xen_intr_pirq_enable_intr(struct intsrc *base_isrc) -{ - struct xenisrc *isrc; - struct evtchn_bind_pirq bind_pirq; - struct physdev_irq_status_query irq_status; - int error; - - isrc = (struct xenisrc *)base_isrc; - - if (!xen_intr_pirq_eoi_map_enabled) { - irq_status.irq = isrc->xi_pirq; - error = HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, - &irq_status); - if (error) - panic("unable to get status of IRQ#%d", isrc->xi_pirq); - - if (irq_status.flags & XENIRQSTAT_needs_eoi) { - /* - * Since the dynamic PIRQ EOI map is not available - * mark the PIRQ as needing EOI unconditionally. - */ - set_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map); - } - } - - bind_pirq.pirq = isrc->xi_pirq; - bind_pirq.flags = isrc->xi_edgetrigger ? 0 : BIND_PIRQ__WILL_SHARE; - error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq); - if (error) - panic("unable to bind IRQ#%d", isrc->xi_pirq); - - isrc->xi_port = bind_pirq.port; - - mtx_lock(&xen_intr_isrc_lock); - KASSERT((xen_intr_port_to_isrc[bind_pirq.port] == NULL), - ("trying to override an already setup event channel port")); - xen_intr_port_to_isrc[bind_pirq.port] = isrc; - mtx_unlock(&xen_intr_isrc_lock); - - evtchn_unmask_port(isrc->xi_port); -} - -/* - * Disable an interrupt source. - * - * \param isrc The interrupt source to disable. - */ -static void -xen_intr_pirq_disable_intr(struct intsrc *base_isrc) -{ - struct xenisrc *isrc; - struct evtchn_close close; - int error; - - isrc = (struct xenisrc *)base_isrc; - - evtchn_mask_port(isrc->xi_port); - - close.port = isrc->xi_port; - error = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close); - if (error) - panic("unable to close event channel %d IRQ#%d", - isrc->xi_port, isrc->xi_pirq); - - mtx_lock(&xen_intr_isrc_lock); - xen_intr_port_to_isrc[isrc->xi_port] = NULL; - mtx_unlock(&xen_intr_isrc_lock); - - isrc->xi_port = 0; -} - -/** - * Perform configuration of an interrupt source. - * - * \param isrc The interrupt source to configure. - * \param trig Edge or level. - * \param pol Active high or low. - * - * \returns 0 if no events are pending, otherwise non-zero. - */ -static int -xen_intr_pirq_config_intr(struct intsrc *base_isrc, enum intr_trigger trig, - enum intr_polarity pol) -{ - struct xenisrc *isrc = (struct xenisrc *)base_isrc; - struct physdev_setup_gsi setup_gsi; - int error; - - KASSERT(!(trig == INTR_TRIGGER_CONFORM || pol == INTR_POLARITY_CONFORM), - ("%s: Conforming trigger or polarity\n", __func__)); - - setup_gsi.gsi = isrc->xi_pirq; - setup_gsi.triggering = trig == INTR_TRIGGER_EDGE ? 0 : 1; - setup_gsi.polarity = pol == INTR_POLARITY_HIGH ? 0 : 1; - - error = HYPERVISOR_physdev_op(PHYSDEVOP_setup_gsi, &setup_gsi); - if (error == -XEN_EEXIST) { - if ((isrc->xi_edgetrigger && (trig != INTR_TRIGGER_EDGE)) || - (isrc->xi_activehi && (pol != INTR_POLARITY_HIGH))) - panic("unable to reconfigure interrupt IRQ#%d", - isrc->xi_pirq); - error = 0; - } - if (error) - panic("unable to configure IRQ#%d\n", isrc->xi_pirq); - - isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0; - isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0; - - return (0); -} - -/*--------------------------- Public Functions -------------------------------*/ -/*------- API comments for these methods can be found in xen/xenintr.h -------*/ -int -xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port, - driver_filter_t filter, driver_intr_t handler, void *arg, - enum intr_type flags, xen_intr_handle_t *port_handlep) -{ - struct xenisrc *isrc; - int error; - - error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, dev, - filter, handler, arg, flags, port_handlep); - if (error != 0) - return (error); - - /* - * By default all interrupts are assigned to vCPU#0 - * unless specified otherwise, so shuffle them to balance - * the interrupt load. - */ - xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu()); - - /* - * The Event Channel API didn't open this port, so it is not - * responsible for closing it automatically on unbind. - */ - isrc->xi_close = 0; - return (0); -} - -int -xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain, - driver_filter_t filter, driver_intr_t handler, void *arg, - enum intr_type flags, xen_intr_handle_t *port_handlep) -{ - struct xenisrc *isrc; - struct evtchn_alloc_unbound alloc_unbound; - int error; - - alloc_unbound.dom = DOMID_SELF; - alloc_unbound.remote_dom = remote_domain; - error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, - &alloc_unbound); - if (error != 0) { - /* - * XXX Trap Hypercall error code Linuxisms in - * the HYPERCALL layer. - */ - return (-error); - } - - error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT, - dev, filter, handler, arg, flags, - port_handlep); - if (error != 0) { - evtchn_close_t close = { .port = alloc_unbound.port }; - if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) - panic("EVTCHNOP_close failed"); - return (error); - } - - /* - * By default all interrupts are assigned to vCPU#0 - * unless specified otherwise, so shuffle them to balance - * the interrupt load. - */ - xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu()); - - isrc->xi_close = 1; - return (0); -} - -int -xen_intr_bind_remote_port(device_t dev, u_int remote_domain, - u_int remote_port, driver_filter_t filter, driver_intr_t handler, - void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) -{ - struct xenisrc *isrc; - struct evtchn_bind_interdomain bind_interdomain; - int error; - - bind_interdomain.remote_dom = remote_domain; - bind_interdomain.remote_port = remote_port; - error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, - &bind_interdomain); - if (error != 0) { - /* - * XXX Trap Hypercall error code Linuxisms in - * the HYPERCALL layer. - */ - return (-error); - } - - error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port, - EVTCHN_TYPE_PORT, dev, filter, handler, - arg, flags, port_handlep); - if (error) { - evtchn_close_t close = { .port = bind_interdomain.local_port }; - if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) - panic("EVTCHNOP_close failed"); - return (error); - } - - /* - * By default all interrupts are assigned to vCPU#0 - * unless specified otherwise, so shuffle them to balance - * the interrupt load. - */ - xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu()); - - /* - * The Event Channel API opened this port, so it is - * responsible for closing it automatically on unbind. - */ - isrc->xi_close = 1; - return (0); -} - -int -xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu, - driver_filter_t filter, driver_intr_t handler, void *arg, - enum intr_type flags, xen_intr_handle_t *port_handlep) -{ - int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; - struct xenisrc *isrc; - struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id }; - int error; - - /* Ensure the target CPU is ready to handle evtchn interrupts. */ - xen_intr_intrcnt_add(cpu); - - isrc = NULL; - error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq); - if (error != 0) { - /* - * XXX Trap Hypercall error code Linuxisms in - * the HYPERCALL layer. - */ - return (-error); - } - - error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, dev, - filter, handler, arg, flags, port_handlep); - -#ifdef SMP - if (error == 0) - error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); -#endif - - if (error != 0) { - evtchn_close_t close = { .port = bind_virq.port }; - - xen_intr_unbind(*port_handlep); - if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) - panic("EVTCHNOP_close failed"); - return (error); - } - -#ifdef SMP - if (isrc->xi_cpu != cpu) { - /* - * Too early in the boot process for the generic interrupt - * code to perform the binding. Update our event channel - * masks manually so events can't fire on the wrong cpu - * during AP startup. - */ - xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]); - } -#endif - - /* - * The Event Channel API opened this port, so it is - * responsible for closing it automatically on unbind. - */ - isrc->xi_close = 1; - isrc->xi_virq = virq; - - return (0); -} - -int -xen_intr_alloc_and_bind_ipi(device_t dev, u_int cpu, - driver_filter_t filter, enum intr_type flags, - xen_intr_handle_t *port_handlep) -{ -#ifdef SMP - int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; - struct xenisrc *isrc; - struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; - int error; - - /* Ensure the target CPU is ready to handle evtchn interrupts. */ - xen_intr_intrcnt_add(cpu); - - isrc = NULL; - error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi); - if (error != 0) { - /* - * XXX Trap Hypercall error code Linuxisms in - * the HYPERCALL layer. - */ - return (-error); - } - - error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI, - dev, filter, NULL, NULL, flags, - port_handlep); - if (error == 0) - error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); - - if (error != 0) { - evtchn_close_t close = { .port = bind_ipi.port }; - - xen_intr_unbind(*port_handlep); - if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) - panic("EVTCHNOP_close failed"); - return (error); - } - - if (isrc->xi_cpu != cpu) { - /* - * Too early in the boot process for the generic interrupt - * code to perform the binding. Update our event channel - * masks manually so events can't fire on the wrong cpu - * during AP startup. - */ - xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]); - } - - /* - * The Event Channel API opened this port, so it is - * responsible for closing it automatically on unbind. - */ - isrc->xi_close = 1; - return (0); -#else - return (EOPNOTSUPP); -#endif -} - -int -xen_register_pirq(int vector, enum intr_trigger trig, enum intr_polarity pol) -{ - struct physdev_map_pirq map_pirq; - struct physdev_irq alloc_pirq; - struct xenisrc *isrc; - int error; - - if (vector == 0) - return (EINVAL); - - if (bootverbose) - printf("xen: register IRQ#%d\n", vector); - - map_pirq.domid = DOMID_SELF; - map_pirq.type = MAP_PIRQ_TYPE_GSI; - map_pirq.index = vector; - map_pirq.pirq = vector; - - error = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_pirq); - if (error) { - printf("xen: unable to map IRQ#%d\n", vector); - return (error); - } - - alloc_pirq.irq = vector; - alloc_pirq.vector = 0; - error = HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &alloc_pirq); - if (error) { - printf("xen: unable to alloc PIRQ for IRQ#%d\n", vector); - return (error); - } - - mtx_lock(&xen_intr_isrc_lock); - isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector); - mtx_unlock(&xen_intr_isrc_lock); - KASSERT((isrc != NULL), ("xen: unable to allocate isrc for interrupt")); - isrc->xi_pirq = vector; - isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0; - isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0; - - return (0); -} - -int -xen_register_msi(device_t dev, int vector, int count) -{ - struct physdev_map_pirq msi_irq; - struct xenisrc *isrc; - int ret; - - memset(&msi_irq, 0, sizeof(msi_irq)); - msi_irq.domid = DOMID_SELF; - msi_irq.type = count == 1 ? - MAP_PIRQ_TYPE_MSI_SEG : MAP_PIRQ_TYPE_MULTI_MSI; - msi_irq.index = -1; - msi_irq.pirq = -1; - msi_irq.bus = pci_get_bus(dev) | (pci_get_domain(dev) << 16); - msi_irq.devfn = (pci_get_slot(dev) << 3) | pci_get_function(dev); - msi_irq.entry_nr = count; - - ret = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &msi_irq); - if (ret != 0) - return (ret); - if (count != msi_irq.entry_nr) { - panic("unable to setup all requested MSI vectors " - "(expected %d got %d)", count, msi_irq.entry_nr); - } - - mtx_lock(&xen_intr_isrc_lock); - for (int i = 0; i < count; i++) { - isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector + i); - KASSERT(isrc != NULL, - ("xen: unable to allocate isrc for interrupt")); - isrc->xi_pirq = msi_irq.pirq + i; - } - mtx_unlock(&xen_intr_isrc_lock); - - return (0); -} - -int -xen_release_msi(int vector) -{ - struct physdev_unmap_pirq unmap; - struct xenisrc *isrc; - int ret; - - isrc = (struct xenisrc *)intr_lookup_source(vector); - if (isrc == NULL) - return (ENXIO); - - unmap.pirq = isrc->xi_pirq; - ret = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap); - if (ret != 0) - return (ret); - - xen_intr_release_isrc(isrc); - - return (0); -} - -int -xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...) -{ - char descr[MAXCOMLEN + 1]; - struct xenisrc *isrc; - va_list ap; - - isrc = xen_intr_isrc(port_handle); - if (isrc == NULL) - return (EINVAL); - - va_start(ap, fmt); - vsnprintf(descr, sizeof(descr), fmt, ap); - va_end(ap); - return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr)); -} - -void -xen_intr_unbind(xen_intr_handle_t *port_handlep) -{ - struct xenisrc *isrc; - - KASSERT(port_handlep != NULL, - ("NULL xen_intr_handle_t passed to xen_intr_unbind")); - - isrc = xen_intr_isrc(*port_handlep); - *port_handlep = NULL; - if (isrc == NULL) - return; - - if (isrc->xi_cookie != NULL) - intr_remove_handler(isrc->xi_cookie); - xen_intr_release_isrc(isrc); -} - -void -xen_intr_signal(xen_intr_handle_t handle) -{ - struct xenisrc *isrc; - - isrc = xen_intr_isrc(handle); - if (isrc != NULL) { - KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT || - isrc->xi_type == EVTCHN_TYPE_IPI, - ("evtchn_signal on something other than a local port")); - struct evtchn_send send = { .port = isrc->xi_port }; - (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send); - } -} - -evtchn_port_t -xen_intr_port(xen_intr_handle_t handle) -{ - struct xenisrc *isrc; - - isrc = xen_intr_isrc(handle); - if (isrc == NULL) - return (0); - - return (isrc->xi_port); -} - -int -xen_intr_add_handler(device_t dev, driver_filter_t filter, - driver_intr_t handler, void *arg, enum intr_type flags, - xen_intr_handle_t handle) -{ - struct xenisrc *isrc; - int error; - - isrc = xen_intr_isrc(handle); - if (isrc == NULL || isrc->xi_cookie != NULL) - return (EINVAL); - - error = intr_add_handler(device_get_nameunit(dev), isrc->xi_vector, - filter, handler, arg, flags|INTR_EXCL, &isrc->xi_cookie); - if (error != 0) { - device_printf(dev, - "xen_intr_add_handler: intr_add_handler failed: %d\n", - error); - } - - return (error); -} - -#ifdef DDB -static const char * -xen_intr_print_type(enum evtchn_type type) -{ - static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = { - [EVTCHN_TYPE_UNBOUND] = "UNBOUND", - [EVTCHN_TYPE_PIRQ] = "PIRQ", - [EVTCHN_TYPE_VIRQ] = "VIRQ", - [EVTCHN_TYPE_IPI] = "IPI", - [EVTCHN_TYPE_PORT] = "PORT", - }; - - if (type >= EVTCHN_TYPE_COUNT) - return ("UNKNOWN"); - - return (evtchn_type_to_string[type]); -} - -static void -xen_intr_dump_port(struct xenisrc *isrc) -{ - struct xen_intr_pcpu_data *pcpu; - shared_info_t *s = HYPERVISOR_shared_info; - int i; - - db_printf("Port %d Type: %s\n", - isrc->xi_port, xen_intr_print_type(isrc->xi_type)); - if (isrc->xi_type == EVTCHN_TYPE_PIRQ) { - db_printf("\tPirq: %d ActiveHi: %d EdgeTrigger: %d " - "NeedsEOI: %d Shared: %d\n", - isrc->xi_pirq, isrc->xi_activehi, isrc->xi_edgetrigger, - !!test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map), - isrc->xi_shared); - } - if (isrc->xi_type == EVTCHN_TYPE_VIRQ) - db_printf("\tVirq: %d\n", isrc->xi_virq); - - db_printf("\tMasked: %d Pending: %d\n", - !!test_bit(isrc->xi_port, &s->evtchn_mask[0]), - !!test_bit(isrc->xi_port, &s->evtchn_pending[0])); - - db_printf("\tPer-CPU Masks: "); - CPU_FOREACH(i) { - pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); - db_printf("cpu#%d: %d ", i, - !!test_bit(isrc->xi_port, pcpu->evtchn_enabled)); - } - db_printf("\n"); -} - -DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn) -{ - int i; - - if (!xen_domain()) { - db_printf("Only available on Xen guests\n"); - return; - } - - for (i = 0; i < NR_EVENT_CHANNELS; i++) { - struct xenisrc *isrc; - - isrc = xen_intr_port_to_isrc[i]; - if (isrc == NULL) - continue; - - xen_intr_dump_port(isrc); - } -} -#endif /* DDB */ diff --git a/sys/xen/xen_intr.c b/sys/xen/xen_intr.c new file mode 100644 index 0000000..3bd292e --- /dev/null +++ b/sys/xen/xen_intr.c @@ -0,0 +1,1626 @@ +/****************************************************************************** + * xen_intr.c + * + * Xen event and interrupt services for x86 PV and HVM guests. + * + * Copyright (c) 2002-2005, K A Fraser + * Copyright (c) 2005, Intel Corporation + * Copyright (c) 2012, Spectra Logic Corporation + * + * This file may be distributed separately from the Linux kernel, or + * incorporated into other software packages, subject to the following license: + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this source file (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_ddb.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include + +#ifdef DDB +#include +#endif + +static MALLOC_DEFINE(M_XENINTR, "xen_intr", "Xen Interrupt Services"); + +/** + * Per-cpu event channel processing state. + */ +struct xen_intr_pcpu_data { + /** + * The last event channel bitmap section (level one bit) processed. + * This is used to ensure we scan all ports before + * servicing an already servied port again. + */ + u_int last_processed_l1i; + + /** + * The last event channel processed within the event channel + * bitmap being scanned. + */ + u_int last_processed_l2i; + + /** Pointer to this CPU's interrupt statistic counter. */ + u_long *evtchn_intrcnt; + + /** + * A bitmap of ports that can be serviced from this CPU. + * A set bit means interrupt handling is enabled. + */ + u_long evtchn_enabled[sizeof(u_long) * 8]; +}; + +/* + * Start the scan at port 0 by initializing the last scanned + * location as the highest numbered event channel port. + */ +DPCPU_DEFINE(struct xen_intr_pcpu_data, xen_intr_pcpu) = { + .last_processed_l1i = LONG_BIT - 1, + .last_processed_l2i = LONG_BIT - 1 +}; + +DPCPU_DECLARE(struct vcpu_info *, vcpu_info); + +#define XEN_EEXIST 17 /* Xen "already exists" error */ +#define XEN_ALLOCATE_VECTOR 0 /* Allocate a vector for this event channel */ +#define XEN_INVALID_EVTCHN 0 /* Invalid event channel */ + +#define is_valid_evtchn(x) ((x) != XEN_INVALID_EVTCHN) + +struct xenisrc { + struct intsrc xi_intsrc; + enum evtchn_type xi_type; + int xi_cpu; /* VCPU for delivery. */ + int xi_vector; /* Global isrc vector number. */ + evtchn_port_t xi_port; + int xi_pirq; + int xi_virq; + void *xi_cookie; + u_int xi_close:1; /* close on unbind? */ + u_int xi_shared:1; /* Shared with other domains. */ + u_int xi_activehi:1; + u_int xi_edgetrigger:1; +}; + +#define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) + +static void xen_intr_suspend(struct pic *); +static void xen_intr_resume(struct pic *, bool suspend_cancelled); +static void xen_intr_enable_source(struct intsrc *isrc); +static void xen_intr_disable_source(struct intsrc *isrc, int eoi); +static void xen_intr_eoi_source(struct intsrc *isrc); +static void xen_intr_enable_intr(struct intsrc *isrc); +static void xen_intr_disable_intr(struct intsrc *isrc); +static int xen_intr_vector(struct intsrc *isrc); +static int xen_intr_source_pending(struct intsrc *isrc); +static int xen_intr_config_intr(struct intsrc *isrc, + enum intr_trigger trig, enum intr_polarity pol); +static int xen_intr_assign_cpu(struct intsrc *isrc, u_int apic_id); + +static void xen_intr_pirq_enable_source(struct intsrc *isrc); +static void xen_intr_pirq_disable_source(struct intsrc *isrc, int eoi); +static void xen_intr_pirq_eoi_source(struct intsrc *isrc); +static void xen_intr_pirq_enable_intr(struct intsrc *isrc); +static void xen_intr_pirq_disable_intr(struct intsrc *isrc); +static int xen_intr_pirq_config_intr(struct intsrc *isrc, + enum intr_trigger trig, enum intr_polarity pol); + +/** + * PIC interface for all event channel port types except physical IRQs. + */ +struct pic xen_intr_pic = { + .pic_enable_source = xen_intr_enable_source, + .pic_disable_source = xen_intr_disable_source, + .pic_eoi_source = xen_intr_eoi_source, + .pic_enable_intr = xen_intr_enable_intr, + .pic_disable_intr = xen_intr_disable_intr, + .pic_vector = xen_intr_vector, + .pic_source_pending = xen_intr_source_pending, + .pic_suspend = xen_intr_suspend, + .pic_resume = xen_intr_resume, + .pic_config_intr = xen_intr_config_intr, + .pic_assign_cpu = xen_intr_assign_cpu +}; + +/** + * PIC interface for all event channel representing + * physical interrupt sources. + */ +struct pic xen_intr_pirq_pic = { + .pic_enable_source = xen_intr_pirq_enable_source, + .pic_disable_source = xen_intr_pirq_disable_source, + .pic_eoi_source = xen_intr_pirq_eoi_source, + .pic_enable_intr = xen_intr_pirq_enable_intr, + .pic_disable_intr = xen_intr_pirq_disable_intr, + .pic_vector = xen_intr_vector, + .pic_source_pending = xen_intr_source_pending, + .pic_config_intr = xen_intr_pirq_config_intr, + .pic_assign_cpu = xen_intr_assign_cpu +}; + +static struct mtx xen_intr_isrc_lock; +static int xen_intr_auto_vector_count; +static struct xenisrc *xen_intr_port_to_isrc[NR_EVENT_CHANNELS]; +static u_long *xen_intr_pirq_eoi_map; +static boolean_t xen_intr_pirq_eoi_map_enabled; + +/*------------------------- Private Functions --------------------------------*/ +/** + * Disable signal delivery for an event channel port on the + * specified CPU. + * + * \param port The event channel port to mask. + * + * This API is used to manage the port<=>CPU binding of event + * channel handlers. + * + * \note This operation does not preclude reception of an event + * for this event channel on another CPU. To mask the + * event channel globally, use evtchn_mask(). + */ +static inline void +evtchn_cpu_mask_port(u_int cpu, evtchn_port_t port) +{ + struct xen_intr_pcpu_data *pcpu; + + pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); + clear_bit(port, pcpu->evtchn_enabled); +} + +/** + * Enable signal delivery for an event channel port on the + * specified CPU. + * + * \param port The event channel port to unmask. + * + * This API is used to manage the port<=>CPU binding of event + * channel handlers. + * + * \note This operation does not guarantee that event delivery + * is enabled for this event channel port. The port must + * also be globally enabled. See evtchn_unmask(). + */ +static inline void +evtchn_cpu_unmask_port(u_int cpu, evtchn_port_t port) +{ + struct xen_intr_pcpu_data *pcpu; + + pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); + set_bit(port, pcpu->evtchn_enabled); +} + +/** + * Allocate and register a per-cpu Xen upcall interrupt counter. + * + * \param cpu The cpu for which to register this interrupt count. + */ +static void +xen_intr_intrcnt_add(u_int cpu) +{ + char buf[MAXCOMLEN + 1]; + struct xen_intr_pcpu_data *pcpu; + + pcpu = DPCPU_ID_PTR(cpu, xen_intr_pcpu); + if (pcpu->evtchn_intrcnt != NULL) + return; + + snprintf(buf, sizeof(buf), "cpu%d:xen", cpu); + intrcnt_add(buf, &pcpu->evtchn_intrcnt); +} + +/** + * Search for an already allocated but currently unused Xen interrupt + * source object. + * + * \param type Restrict the search to interrupt sources of the given + * type. + * + * \return A pointer to a free Xen interrupt source object or NULL. + */ +static struct xenisrc * +xen_intr_find_unused_isrc(enum evtchn_type type) +{ + int isrc_idx; + + KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn isrc lock not held")); + + for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx ++) { + struct xenisrc *isrc; + u_int vector; + + vector = FIRST_EVTCHN_INT + isrc_idx; + isrc = (struct xenisrc *)intr_lookup_source(vector); + if (isrc != NULL + && isrc->xi_type == EVTCHN_TYPE_UNBOUND) { + KASSERT(isrc->xi_intsrc.is_handlers == 0, + ("Free evtchn still has handlers")); + isrc->xi_type = type; + return (isrc); + } + } + return (NULL); +} + +/** + * Allocate a Xen interrupt source object. + * + * \param type The type of interrupt source to create. + * + * \return A pointer to a newly allocated Xen interrupt source + * object or NULL. + */ +static struct xenisrc * +xen_intr_alloc_isrc(enum evtchn_type type, int vector) +{ + static int warned; + struct xenisrc *isrc; + + KASSERT(mtx_owned(&xen_intr_isrc_lock), ("Evtchn alloc lock not held")); + + if (xen_intr_auto_vector_count > NR_EVENT_CHANNELS) { + if (!warned) { + warned = 1; + printf("xen_intr_alloc: Event channels exhausted.\n"); + } + return (NULL); + } + + if (type != EVTCHN_TYPE_PIRQ) { + vector = FIRST_EVTCHN_INT + xen_intr_auto_vector_count; + xen_intr_auto_vector_count++; + } + + KASSERT((intr_lookup_source(vector) == NULL), + ("Trying to use an already allocated vector")); + + mtx_unlock(&xen_intr_isrc_lock); + isrc = malloc(sizeof(*isrc), M_XENINTR, M_WAITOK | M_ZERO); + isrc->xi_intsrc.is_pic = + (type == EVTCHN_TYPE_PIRQ) ? &xen_intr_pirq_pic : &xen_intr_pic; + isrc->xi_vector = vector; + isrc->xi_type = type; + intr_register_source(&isrc->xi_intsrc); + mtx_lock(&xen_intr_isrc_lock); + + return (isrc); +} + +/** + * Attempt to free an active Xen interrupt source object. + * + * \param isrc The interrupt source object to release. + * + * \returns EBUSY if the source is still in use, otherwise 0. + */ +static int +xen_intr_release_isrc(struct xenisrc *isrc) +{ + + mtx_lock(&xen_intr_isrc_lock); + if (isrc->xi_intsrc.is_handlers != 0) { + mtx_unlock(&xen_intr_isrc_lock); + return (EBUSY); + } + evtchn_mask_port(isrc->xi_port); + evtchn_clear_port(isrc->xi_port); + + /* Rebind port to CPU 0. */ + evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); + evtchn_cpu_unmask_port(0, isrc->xi_port); + + if (isrc->xi_close != 0 && is_valid_evtchn(isrc->xi_port)) { + struct evtchn_close close = { .port = isrc->xi_port }; + if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) + panic("EVTCHNOP_close failed"); + } + + xen_intr_port_to_isrc[isrc->xi_port] = NULL; + isrc->xi_cpu = 0; + isrc->xi_type = EVTCHN_TYPE_UNBOUND; + isrc->xi_port = 0; + isrc->xi_cookie = NULL; + mtx_unlock(&xen_intr_isrc_lock); + return (0); +} + +/** + * Associate an interrupt handler with an already allocated local Xen + * event channel port. + * + * \param isrcp The returned Xen interrupt object associated with + * the specified local port. + * \param local_port The event channel to bind. + * \param type The event channel type of local_port. + * \param intr_owner The device making this bind request. + * \param filter An interrupt filter handler. Specify NULL + * to always dispatch to the ithread handler. + * \param handler An interrupt ithread handler. Optional (can + * specify NULL) if all necessary event actions + * are performed by filter. + * \param arg Argument to present to both filter and handler. + * \param irqflags Interrupt handler flags. See sys/bus.h. + * \param handlep Pointer to an opaque handle used to manage this + * registration. + * + * \returns 0 on success, otherwise an errno. + */ +static int +xen_intr_bind_isrc(struct xenisrc **isrcp, evtchn_port_t local_port, + enum evtchn_type type, device_t intr_owner, driver_filter_t filter, + driver_intr_t handler, void *arg, enum intr_type flags, + xen_intr_handle_t *port_handlep) +{ + struct xenisrc *isrc; + int error; + + *isrcp = NULL; + if (port_handlep == NULL) { + device_printf(intr_owner, + "xen_intr_bind_isrc: Bad event handle\n"); + return (EINVAL); + } + + mtx_lock(&xen_intr_isrc_lock); + isrc = xen_intr_find_unused_isrc(type); + if (isrc == NULL) { + isrc = xen_intr_alloc_isrc(type, XEN_ALLOCATE_VECTOR); + if (isrc == NULL) { + mtx_unlock(&xen_intr_isrc_lock); + return (ENOSPC); + } + } + isrc->xi_port = local_port; + xen_intr_port_to_isrc[local_port] = isrc; + mtx_unlock(&xen_intr_isrc_lock); + + /* Assign the opaque handler (the event channel port) */ + *port_handlep = &isrc->xi_port; + + if (filter == NULL && handler == NULL) { + /* + * No filter/handler provided, leave the event channel + * masked and without a valid handler, the caller is + * in charge of setting that up. + */ + *isrcp = isrc; + return (0); + } + + error = xen_intr_add_handler(intr_owner, filter, handler, arg, flags, + *port_handlep); + if (error != 0) { + xen_intr_release_isrc(isrc); + return (error); + } + *isrcp = isrc; + return (0); +} + +/** + * Lookup a Xen interrupt source object given an interrupt binding handle. + * + * \param handle A handle initialized by a previous call to + * xen_intr_bind_isrc(). + * + * \returns A pointer to the Xen interrupt source object associated + * with the given interrupt handle. NULL if no association + * currently exists. + */ +static struct xenisrc * +xen_intr_isrc(xen_intr_handle_t handle) +{ + evtchn_port_t port; + + if (handle == NULL) + return (NULL); + + port = *(evtchn_port_t *)handle; + if (!is_valid_evtchn(port) || port >= NR_EVENT_CHANNELS) + return (NULL); + + return (xen_intr_port_to_isrc[port]); +} + +/** + * Determine the event channel ports at the given section of the + * event port bitmap which have pending events for the given cpu. + * + * \param pcpu The Xen interrupt pcpu data for the cpu being querried. + * \param sh The Xen shared info area. + * \param idx The index of the section of the event channel bitmap to + * inspect. + * + * \returns A u_long with bits set for every event channel with pending + * events. + */ +static inline u_long +xen_intr_active_ports(struct xen_intr_pcpu_data *pcpu, shared_info_t *sh, + u_int idx) +{ + return (sh->evtchn_pending[idx] + & ~sh->evtchn_mask[idx] + & pcpu->evtchn_enabled[idx]); +} + +/** + * Interrupt handler for processing all Xen event channel events. + * + * \param trap_frame The trap frame context for the current interrupt. + */ +void +xen_intr_handle_upcall(struct trapframe *trap_frame) +{ + u_int l1i, l2i, port, cpu; + u_long masked_l1, masked_l2; + struct xenisrc *isrc; + shared_info_t *s; + vcpu_info_t *v; + struct xen_intr_pcpu_data *pc; + u_long l1, l2; + + /* + * Disable preemption in order to always check and fire events + * on the right vCPU + */ + critical_enter(); + + cpu = PCPU_GET(cpuid); + pc = DPCPU_PTR(xen_intr_pcpu); + s = HYPERVISOR_shared_info; + v = DPCPU_GET(vcpu_info); + + if (xen_hvm_domain() && !xen_vector_callback_enabled) { + KASSERT((cpu == 0), ("Fired PCI event callback on wrong CPU")); + } + + v->evtchn_upcall_pending = 0; + +#if 0 +#ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */ + /* Clear master flag /before/ clearing selector flag. */ + wmb(); +#endif +#endif + + l1 = atomic_readandclear_long(&v->evtchn_pending_sel); + + l1i = pc->last_processed_l1i; + l2i = pc->last_processed_l2i; + (*pc->evtchn_intrcnt)++; + + while (l1 != 0) { + + l1i = (l1i + 1) % LONG_BIT; + masked_l1 = l1 & ((~0UL) << l1i); + + if (masked_l1 == 0) { + /* + * if we masked out all events, wrap around + * to the beginning. + */ + l1i = LONG_BIT - 1; + l2i = LONG_BIT - 1; + continue; + } + l1i = ffsl(masked_l1) - 1; + + do { + l2 = xen_intr_active_ports(pc, s, l1i); + + l2i = (l2i + 1) % LONG_BIT; + masked_l2 = l2 & ((~0UL) << l2i); + + if (masked_l2 == 0) { + /* if we masked out all events, move on */ + l2i = LONG_BIT - 1; + break; + } + l2i = ffsl(masked_l2) - 1; + + /* process port */ + port = (l1i * LONG_BIT) + l2i; + synch_clear_bit(port, &s->evtchn_pending[0]); + + isrc = xen_intr_port_to_isrc[port]; + if (__predict_false(isrc == NULL)) + continue; + + /* Make sure we are firing on the right vCPU */ + KASSERT((isrc->xi_cpu == PCPU_GET(cpuid)), + ("Received unexpected event on vCPU#%d, event bound to vCPU#%d", + PCPU_GET(cpuid), isrc->xi_cpu)); + + intr_execute_handlers(&isrc->xi_intsrc, trap_frame); + + /* + * If this is the final port processed, + * we'll pick up here+1 next time. + */ + pc->last_processed_l1i = l1i; + pc->last_processed_l2i = l2i; + + } while (l2i != LONG_BIT - 1); + + l2 = xen_intr_active_ports(pc, s, l1i); + if (l2 == 0) { + /* + * We handled all ports, so we can clear the + * selector bit. + */ + l1 &= ~(1UL << l1i); + } + } + critical_exit(); +} + +static int +xen_intr_init(void *dummy __unused) +{ + shared_info_t *s = HYPERVISOR_shared_info; + struct xen_intr_pcpu_data *pcpu; + struct physdev_pirq_eoi_gmfn eoi_gmfn; + int i, rc; + + if (!xen_domain()) + return (0); + + mtx_init(&xen_intr_isrc_lock, "xen-irq-lock", NULL, MTX_DEF); + + /* + * Register interrupt count manually as we aren't + * guaranteed to see a call to xen_intr_assign_cpu() + * before our first interrupt. Also set the per-cpu + * mask of CPU#0 to enable all, since by default + * all event channels are bound to CPU#0. + */ + CPU_FOREACH(i) { + pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); + memset(pcpu->evtchn_enabled, i == 0 ? ~0 : 0, + sizeof(pcpu->evtchn_enabled)); + xen_intr_intrcnt_add(i); + } + + for (i = 0; i < nitems(s->evtchn_mask); i++) + atomic_store_rel_long(&s->evtchn_mask[i], ~0); + + /* Try to register PIRQ EOI map */ + xen_intr_pirq_eoi_map = malloc(PAGE_SIZE, M_XENINTR, M_WAITOK | M_ZERO); + eoi_gmfn.gmfn = atop(vtophys(xen_intr_pirq_eoi_map)); + rc = HYPERVISOR_physdev_op(PHYSDEVOP_pirq_eoi_gmfn_v2, &eoi_gmfn); + if (rc != 0 && bootverbose) + printf("Xen interrupts: unable to register PIRQ EOI map\n"); + else + xen_intr_pirq_eoi_map_enabled = true; + + intr_register_pic(&xen_intr_pic); + intr_register_pic(&xen_intr_pirq_pic); + + if (bootverbose) + printf("Xen interrupt system initialized\n"); + + return (0); +} +SYSINIT(xen_intr_init, SI_SUB_INTR, SI_ORDER_SECOND, xen_intr_init, NULL); + +/*--------------------------- Common PIC Functions ---------------------------*/ +/** + * Prepare this PIC for system suspension. + */ +static void +xen_intr_suspend(struct pic *unused) +{ +} + +static void +xen_rebind_ipi(struct xenisrc *isrc) +{ +#ifdef SMP + int cpu = isrc->xi_cpu; + int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; + int error; + struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; + + error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, + &bind_ipi); + if (error != 0) + panic("unable to rebind xen IPI: %d", error); + + isrc->xi_port = bind_ipi.port; + isrc->xi_cpu = 0; + xen_intr_port_to_isrc[bind_ipi.port] = isrc; + + error = xen_intr_assign_cpu(&isrc->xi_intsrc, + cpu_apic_ids[cpu]); + if (error) + panic("unable to bind xen IPI to CPU#%d: %d", + cpu, error); + + evtchn_unmask_port(bind_ipi.port); +#else + panic("Resume IPI event channel on UP"); +#endif +} + +static void +xen_rebind_virq(struct xenisrc *isrc) +{ + int cpu = isrc->xi_cpu; + int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; + int error; + struct evtchn_bind_virq bind_virq = { .virq = isrc->xi_virq, + .vcpu = vcpu_id }; + + error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, + &bind_virq); + if (error != 0) + panic("unable to rebind xen VIRQ#%d: %d", isrc->xi_virq, error); + + isrc->xi_port = bind_virq.port; + isrc->xi_cpu = 0; + xen_intr_port_to_isrc[bind_virq.port] = isrc; + +#ifdef SMP + error = xen_intr_assign_cpu(&isrc->xi_intsrc, + cpu_apic_ids[cpu]); + if (error) + panic("unable to bind xen VIRQ#%d to CPU#%d: %d", + isrc->xi_virq, cpu, error); +#endif + + evtchn_unmask_port(bind_virq.port); +} + +/** + * Return this PIC to service after being suspended. + */ +static void +xen_intr_resume(struct pic *unused, bool suspend_cancelled) +{ + shared_info_t *s = HYPERVISOR_shared_info; + struct xenisrc *isrc; + u_int isrc_idx; + int i; + + if (suspend_cancelled) + return; + + /* Reset the per-CPU masks */ + CPU_FOREACH(i) { + struct xen_intr_pcpu_data *pcpu; + + pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); + memset(pcpu->evtchn_enabled, + i == 0 ? ~0 : 0, sizeof(pcpu->evtchn_enabled)); + } + + /* Mask all event channels. */ + for (i = 0; i < nitems(s->evtchn_mask); i++) + atomic_store_rel_long(&s->evtchn_mask[i], ~0); + + /* Remove port -> isrc mappings */ + memset(xen_intr_port_to_isrc, 0, sizeof(xen_intr_port_to_isrc)); + + /* Free unused isrcs and rebind VIRQs and IPIs */ + for (isrc_idx = 0; isrc_idx < xen_intr_auto_vector_count; isrc_idx++) { + u_int vector; + + vector = FIRST_EVTCHN_INT + isrc_idx; + isrc = (struct xenisrc *)intr_lookup_source(vector); + if (isrc != NULL) { + isrc->xi_port = 0; + switch (isrc->xi_type) { + case EVTCHN_TYPE_IPI: + xen_rebind_ipi(isrc); + break; + case EVTCHN_TYPE_VIRQ: + xen_rebind_virq(isrc); + break; + default: + isrc->xi_cpu = 0; + break; + } + } + } +} + +/** + * Disable a Xen interrupt source. + * + * \param isrc The interrupt source to disable. + */ +static void +xen_intr_disable_intr(struct intsrc *base_isrc) +{ + struct xenisrc *isrc = (struct xenisrc *)base_isrc; + + evtchn_mask_port(isrc->xi_port); +} + +/** + * Determine the global interrupt vector number for + * a Xen interrupt source. + * + * \param isrc The interrupt source to query. + * + * \return The vector number corresponding to the given interrupt source. + */ +static int +xen_intr_vector(struct intsrc *base_isrc) +{ + struct xenisrc *isrc = (struct xenisrc *)base_isrc; + + return (isrc->xi_vector); +} + +/** + * Determine whether or not interrupt events are pending on the + * the given interrupt source. + * + * \param isrc The interrupt source to query. + * + * \returns 0 if no events are pending, otherwise non-zero. + */ +static int +xen_intr_source_pending(struct intsrc *isrc) +{ + /* + * EventChannels are edge triggered and never masked. + * There can be no pending events. + */ + return (0); +} + +/** + * Perform configuration of an interrupt source. + * + * \param isrc The interrupt source to configure. + * \param trig Edge or level. + * \param pol Active high or low. + * + * \returns 0 if no events are pending, otherwise non-zero. + */ +static int +xen_intr_config_intr(struct intsrc *isrc, enum intr_trigger trig, + enum intr_polarity pol) +{ + /* Configuration is only possible via the evtchn apis. */ + return (ENODEV); +} + +/** + * Configure CPU affinity for interrupt source event delivery. + * + * \param isrc The interrupt source to configure. + * \param apic_id The apic id of the CPU for handling future events. + * + * \returns 0 if successful, otherwise an errno. + */ +static int +xen_intr_assign_cpu(struct intsrc *base_isrc, u_int apic_id) +{ +#ifdef SMP + struct evtchn_bind_vcpu bind_vcpu; + struct xenisrc *isrc; + u_int to_cpu, vcpu_id; + int error, masked; + +#ifdef XENHVM + if (xen_vector_callback_enabled == 0) + return (EOPNOTSUPP); +#endif + + to_cpu = apic_cpuid(apic_id); + vcpu_id = pcpu_find(to_cpu)->pc_vcpu_id; + xen_intr_intrcnt_add(to_cpu); + + mtx_lock(&xen_intr_isrc_lock); + isrc = (struct xenisrc *)base_isrc; + if (!is_valid_evtchn(isrc->xi_port)) { + mtx_unlock(&xen_intr_isrc_lock); + return (EINVAL); + } + + /* + * Mask the event channel while binding it to prevent interrupt + * delivery with an inconsistent state in isrc->xi_cpu. + */ + masked = evtchn_test_and_set_mask(isrc->xi_port); + if ((isrc->xi_type == EVTCHN_TYPE_VIRQ) || + (isrc->xi_type == EVTCHN_TYPE_IPI)) { + /* + * Virtual IRQs are associated with a cpu by + * the Hypervisor at evtchn_bind_virq time, so + * all we need to do is update the per-CPU masks. + */ + evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); + isrc->xi_cpu = to_cpu; + evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); + goto out; + } + + bind_vcpu.port = isrc->xi_port; + bind_vcpu.vcpu = vcpu_id; + + error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind_vcpu); + if (isrc->xi_cpu != to_cpu) { + if (error == 0) { + /* Commit to new binding by removing the old one. */ + evtchn_cpu_mask_port(isrc->xi_cpu, isrc->xi_port); + isrc->xi_cpu = to_cpu; + evtchn_cpu_unmask_port(isrc->xi_cpu, isrc->xi_port); + } + } + +out: + if (masked == 0) + evtchn_unmask_port(isrc->xi_port); + mtx_unlock(&xen_intr_isrc_lock); + return (0); +#else + return (EOPNOTSUPP); +#endif +} + +/*------------------- Virtual Interrupt Source PIC Functions -----------------*/ +/* + * Mask a level triggered interrupt source. + * + * \param isrc The interrupt source to mask (if necessary). + * \param eoi If non-zero, perform any necessary end-of-interrupt + * acknowledgements. + */ +static void +xen_intr_disable_source(struct intsrc *isrc, int eoi) +{ +} + +/* + * Unmask a level triggered interrupt source. + * + * \param isrc The interrupt source to unmask (if necessary). + */ +static void +xen_intr_enable_source(struct intsrc *isrc) +{ +} + +/* + * Perform any necessary end-of-interrupt acknowledgements. + * + * \param isrc The interrupt source to EOI. + */ +static void +xen_intr_eoi_source(struct intsrc *isrc) +{ +} + +/* + * Enable and unmask the interrupt source. + * + * \param isrc The interrupt source to enable. + */ +static void +xen_intr_enable_intr(struct intsrc *base_isrc) +{ + struct xenisrc *isrc = (struct xenisrc *)base_isrc; + + evtchn_unmask_port(isrc->xi_port); +} + +/*------------------ Physical Interrupt Source PIC Functions -----------------*/ +/* + * Mask a level triggered interrupt source. + * + * \param isrc The interrupt source to mask (if necessary). + * \param eoi If non-zero, perform any necessary end-of-interrupt + * acknowledgements. + */ +static void +xen_intr_pirq_disable_source(struct intsrc *base_isrc, int eoi) +{ + struct xenisrc *isrc; + + isrc = (struct xenisrc *)base_isrc; + evtchn_mask_port(isrc->xi_port); + + if (eoi == PIC_EOI) + xen_intr_pirq_eoi_source(base_isrc); +} + +/* + * Unmask a level triggered interrupt source. + * + * \param isrc The interrupt source to unmask (if necessary). + */ +static void +xen_intr_pirq_enable_source(struct intsrc *base_isrc) +{ + struct xenisrc *isrc; + + isrc = (struct xenisrc *)base_isrc; + evtchn_unmask_port(isrc->xi_port); +} + +/* + * Perform any necessary end-of-interrupt acknowledgements. + * + * \param isrc The interrupt source to EOI. + */ +static void +xen_intr_pirq_eoi_source(struct intsrc *base_isrc) +{ + struct xenisrc *isrc; + + /* XXX Use shared page of flags for this. */ + isrc = (struct xenisrc *)base_isrc; + if (test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map)) { + struct physdev_eoi eoi = { .irq = isrc->xi_pirq }; + + (void)HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi); + } +} + +/* + * Enable and unmask the interrupt source. + * + * \param isrc The interrupt source to enable. + */ +static void +xen_intr_pirq_enable_intr(struct intsrc *base_isrc) +{ + struct xenisrc *isrc; + struct evtchn_bind_pirq bind_pirq; + struct physdev_irq_status_query irq_status; + int error; + + isrc = (struct xenisrc *)base_isrc; + + if (!xen_intr_pirq_eoi_map_enabled) { + irq_status.irq = isrc->xi_pirq; + error = HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, + &irq_status); + if (error) + panic("unable to get status of IRQ#%d", isrc->xi_pirq); + + if (irq_status.flags & XENIRQSTAT_needs_eoi) { + /* + * Since the dynamic PIRQ EOI map is not available + * mark the PIRQ as needing EOI unconditionally. + */ + set_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map); + } + } + + bind_pirq.pirq = isrc->xi_pirq; + bind_pirq.flags = isrc->xi_edgetrigger ? 0 : BIND_PIRQ__WILL_SHARE; + error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind_pirq); + if (error) + panic("unable to bind IRQ#%d", isrc->xi_pirq); + + isrc->xi_port = bind_pirq.port; + + mtx_lock(&xen_intr_isrc_lock); + KASSERT((xen_intr_port_to_isrc[bind_pirq.port] == NULL), + ("trying to override an already setup event channel port")); + xen_intr_port_to_isrc[bind_pirq.port] = isrc; + mtx_unlock(&xen_intr_isrc_lock); + + evtchn_unmask_port(isrc->xi_port); +} + +/* + * Disable an interrupt source. + * + * \param isrc The interrupt source to disable. + */ +static void +xen_intr_pirq_disable_intr(struct intsrc *base_isrc) +{ + struct xenisrc *isrc; + struct evtchn_close close; + int error; + + isrc = (struct xenisrc *)base_isrc; + + evtchn_mask_port(isrc->xi_port); + + close.port = isrc->xi_port; + error = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close); + if (error) + panic("unable to close event channel %d IRQ#%d", + isrc->xi_port, isrc->xi_pirq); + + mtx_lock(&xen_intr_isrc_lock); + xen_intr_port_to_isrc[isrc->xi_port] = NULL; + mtx_unlock(&xen_intr_isrc_lock); + + isrc->xi_port = 0; +} + +/** + * Perform configuration of an interrupt source. + * + * \param isrc The interrupt source to configure. + * \param trig Edge or level. + * \param pol Active high or low. + * + * \returns 0 if no events are pending, otherwise non-zero. + */ +static int +xen_intr_pirq_config_intr(struct intsrc *base_isrc, enum intr_trigger trig, + enum intr_polarity pol) +{ + struct xenisrc *isrc = (struct xenisrc *)base_isrc; + struct physdev_setup_gsi setup_gsi; + int error; + + KASSERT(!(trig == INTR_TRIGGER_CONFORM || pol == INTR_POLARITY_CONFORM), + ("%s: Conforming trigger or polarity\n", __func__)); + + setup_gsi.gsi = isrc->xi_pirq; + setup_gsi.triggering = trig == INTR_TRIGGER_EDGE ? 0 : 1; + setup_gsi.polarity = pol == INTR_POLARITY_HIGH ? 0 : 1; + + error = HYPERVISOR_physdev_op(PHYSDEVOP_setup_gsi, &setup_gsi); + if (error == -XEN_EEXIST) { + if ((isrc->xi_edgetrigger && (trig != INTR_TRIGGER_EDGE)) || + (isrc->xi_activehi && (pol != INTR_POLARITY_HIGH))) + panic("unable to reconfigure interrupt IRQ#%d", + isrc->xi_pirq); + error = 0; + } + if (error) + panic("unable to configure IRQ#%d\n", isrc->xi_pirq); + + isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0; + isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0; + + return (0); +} + +/*--------------------------- Public Functions -------------------------------*/ +/*------- API comments for these methods can be found in xen/xenintr.h -------*/ +int +xen_intr_bind_local_port(device_t dev, evtchn_port_t local_port, + driver_filter_t filter, driver_intr_t handler, void *arg, + enum intr_type flags, xen_intr_handle_t *port_handlep) +{ + struct xenisrc *isrc; + int error; + + error = xen_intr_bind_isrc(&isrc, local_port, EVTCHN_TYPE_PORT, dev, + filter, handler, arg, flags, port_handlep); + if (error != 0) + return (error); + + /* + * By default all interrupts are assigned to vCPU#0 + * unless specified otherwise, so shuffle them to balance + * the interrupt load. + */ + xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu()); + + /* + * The Event Channel API didn't open this port, so it is not + * responsible for closing it automatically on unbind. + */ + isrc->xi_close = 0; + return (0); +} + +int +xen_intr_alloc_and_bind_local_port(device_t dev, u_int remote_domain, + driver_filter_t filter, driver_intr_t handler, void *arg, + enum intr_type flags, xen_intr_handle_t *port_handlep) +{ + struct xenisrc *isrc; + struct evtchn_alloc_unbound alloc_unbound; + int error; + + alloc_unbound.dom = DOMID_SELF; + alloc_unbound.remote_dom = remote_domain; + error = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound, + &alloc_unbound); + if (error != 0) { + /* + * XXX Trap Hypercall error code Linuxisms in + * the HYPERCALL layer. + */ + return (-error); + } + + error = xen_intr_bind_isrc(&isrc, alloc_unbound.port, EVTCHN_TYPE_PORT, + dev, filter, handler, arg, flags, + port_handlep); + if (error != 0) { + evtchn_close_t close = { .port = alloc_unbound.port }; + if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) + panic("EVTCHNOP_close failed"); + return (error); + } + + /* + * By default all interrupts are assigned to vCPU#0 + * unless specified otherwise, so shuffle them to balance + * the interrupt load. + */ + xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu()); + + isrc->xi_close = 1; + return (0); +} + +int +xen_intr_bind_remote_port(device_t dev, u_int remote_domain, + u_int remote_port, driver_filter_t filter, driver_intr_t handler, + void *arg, enum intr_type flags, xen_intr_handle_t *port_handlep) +{ + struct xenisrc *isrc; + struct evtchn_bind_interdomain bind_interdomain; + int error; + + bind_interdomain.remote_dom = remote_domain; + bind_interdomain.remote_port = remote_port; + error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, + &bind_interdomain); + if (error != 0) { + /* + * XXX Trap Hypercall error code Linuxisms in + * the HYPERCALL layer. + */ + return (-error); + } + + error = xen_intr_bind_isrc(&isrc, bind_interdomain.local_port, + EVTCHN_TYPE_PORT, dev, filter, handler, + arg, flags, port_handlep); + if (error) { + evtchn_close_t close = { .port = bind_interdomain.local_port }; + if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) + panic("EVTCHNOP_close failed"); + return (error); + } + + /* + * By default all interrupts are assigned to vCPU#0 + * unless specified otherwise, so shuffle them to balance + * the interrupt load. + */ + xen_intr_assign_cpu(&isrc->xi_intsrc, intr_next_cpu()); + + /* + * The Event Channel API opened this port, so it is + * responsible for closing it automatically on unbind. + */ + isrc->xi_close = 1; + return (0); +} + +int +xen_intr_bind_virq(device_t dev, u_int virq, u_int cpu, + driver_filter_t filter, driver_intr_t handler, void *arg, + enum intr_type flags, xen_intr_handle_t *port_handlep) +{ + int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; + struct xenisrc *isrc; + struct evtchn_bind_virq bind_virq = { .virq = virq, .vcpu = vcpu_id }; + int error; + + /* Ensure the target CPU is ready to handle evtchn interrupts. */ + xen_intr_intrcnt_add(cpu); + + isrc = NULL; + error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind_virq); + if (error != 0) { + /* + * XXX Trap Hypercall error code Linuxisms in + * the HYPERCALL layer. + */ + return (-error); + } + + error = xen_intr_bind_isrc(&isrc, bind_virq.port, EVTCHN_TYPE_VIRQ, dev, + filter, handler, arg, flags, port_handlep); + +#ifdef SMP + if (error == 0) + error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); +#endif + + if (error != 0) { + evtchn_close_t close = { .port = bind_virq.port }; + + xen_intr_unbind(*port_handlep); + if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) + panic("EVTCHNOP_close failed"); + return (error); + } + +#ifdef SMP + if (isrc->xi_cpu != cpu) { + /* + * Too early in the boot process for the generic interrupt + * code to perform the binding. Update our event channel + * masks manually so events can't fire on the wrong cpu + * during AP startup. + */ + xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]); + } +#endif + + /* + * The Event Channel API opened this port, so it is + * responsible for closing it automatically on unbind. + */ + isrc->xi_close = 1; + isrc->xi_virq = virq; + + return (0); +} + +int +xen_intr_alloc_and_bind_ipi(device_t dev, u_int cpu, + driver_filter_t filter, enum intr_type flags, + xen_intr_handle_t *port_handlep) +{ +#ifdef SMP + int vcpu_id = pcpu_find(cpu)->pc_vcpu_id; + struct xenisrc *isrc; + struct evtchn_bind_ipi bind_ipi = { .vcpu = vcpu_id }; + int error; + + /* Ensure the target CPU is ready to handle evtchn interrupts. */ + xen_intr_intrcnt_add(cpu); + + isrc = NULL; + error = HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind_ipi); + if (error != 0) { + /* + * XXX Trap Hypercall error code Linuxisms in + * the HYPERCALL layer. + */ + return (-error); + } + + error = xen_intr_bind_isrc(&isrc, bind_ipi.port, EVTCHN_TYPE_IPI, + dev, filter, NULL, NULL, flags, + port_handlep); + if (error == 0) + error = intr_event_bind(isrc->xi_intsrc.is_event, cpu); + + if (error != 0) { + evtchn_close_t close = { .port = bind_ipi.port }; + + xen_intr_unbind(*port_handlep); + if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close)) + panic("EVTCHNOP_close failed"); + return (error); + } + + if (isrc->xi_cpu != cpu) { + /* + * Too early in the boot process for the generic interrupt + * code to perform the binding. Update our event channel + * masks manually so events can't fire on the wrong cpu + * during AP startup. + */ + xen_intr_assign_cpu(&isrc->xi_intsrc, cpu_apic_ids[cpu]); + } + + /* + * The Event Channel API opened this port, so it is + * responsible for closing it automatically on unbind. + */ + isrc->xi_close = 1; + return (0); +#else + return (EOPNOTSUPP); +#endif +} + +int +xen_register_pirq(int vector, enum intr_trigger trig, enum intr_polarity pol) +{ + struct physdev_map_pirq map_pirq; + struct physdev_irq alloc_pirq; + struct xenisrc *isrc; + int error; + + if (vector == 0) + return (EINVAL); + + if (bootverbose) + printf("xen: register IRQ#%d\n", vector); + + map_pirq.domid = DOMID_SELF; + map_pirq.type = MAP_PIRQ_TYPE_GSI; + map_pirq.index = vector; + map_pirq.pirq = vector; + + error = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_pirq); + if (error) { + printf("xen: unable to map IRQ#%d\n", vector); + return (error); + } + + alloc_pirq.irq = vector; + alloc_pirq.vector = 0; + error = HYPERVISOR_physdev_op(PHYSDEVOP_alloc_irq_vector, &alloc_pirq); + if (error) { + printf("xen: unable to alloc PIRQ for IRQ#%d\n", vector); + return (error); + } + + mtx_lock(&xen_intr_isrc_lock); + isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector); + mtx_unlock(&xen_intr_isrc_lock); + KASSERT((isrc != NULL), ("xen: unable to allocate isrc for interrupt")); + isrc->xi_pirq = vector; + isrc->xi_activehi = pol == INTR_POLARITY_HIGH ? 1 : 0; + isrc->xi_edgetrigger = trig == INTR_TRIGGER_EDGE ? 1 : 0; + + return (0); +} + +int +xen_register_msi(device_t dev, int vector, int count) +{ + struct physdev_map_pirq msi_irq; + struct xenisrc *isrc; + int ret; + + memset(&msi_irq, 0, sizeof(msi_irq)); + msi_irq.domid = DOMID_SELF; + msi_irq.type = count == 1 ? + MAP_PIRQ_TYPE_MSI_SEG : MAP_PIRQ_TYPE_MULTI_MSI; + msi_irq.index = -1; + msi_irq.pirq = -1; + msi_irq.bus = pci_get_bus(dev) | (pci_get_domain(dev) << 16); + msi_irq.devfn = (pci_get_slot(dev) << 3) | pci_get_function(dev); + msi_irq.entry_nr = count; + + ret = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &msi_irq); + if (ret != 0) + return (ret); + if (count != msi_irq.entry_nr) { + panic("unable to setup all requested MSI vectors " + "(expected %d got %d)", count, msi_irq.entry_nr); + } + + mtx_lock(&xen_intr_isrc_lock); + for (int i = 0; i < count; i++) { + isrc = xen_intr_alloc_isrc(EVTCHN_TYPE_PIRQ, vector + i); + KASSERT(isrc != NULL, + ("xen: unable to allocate isrc for interrupt")); + isrc->xi_pirq = msi_irq.pirq + i; + } + mtx_unlock(&xen_intr_isrc_lock); + + return (0); +} + +int +xen_release_msi(int vector) +{ + struct physdev_unmap_pirq unmap; + struct xenisrc *isrc; + int ret; + + isrc = (struct xenisrc *)intr_lookup_source(vector); + if (isrc == NULL) + return (ENXIO); + + unmap.pirq = isrc->xi_pirq; + ret = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap); + if (ret != 0) + return (ret); + + xen_intr_release_isrc(isrc); + + return (0); +} + +int +xen_intr_describe(xen_intr_handle_t port_handle, const char *fmt, ...) +{ + char descr[MAXCOMLEN + 1]; + struct xenisrc *isrc; + va_list ap; + + isrc = xen_intr_isrc(port_handle); + if (isrc == NULL) + return (EINVAL); + + va_start(ap, fmt); + vsnprintf(descr, sizeof(descr), fmt, ap); + va_end(ap); + return (intr_describe(isrc->xi_vector, isrc->xi_cookie, descr)); +} + +void +xen_intr_unbind(xen_intr_handle_t *port_handlep) +{ + struct xenisrc *isrc; + + KASSERT(port_handlep != NULL, + ("NULL xen_intr_handle_t passed to xen_intr_unbind")); + + isrc = xen_intr_isrc(*port_handlep); + *port_handlep = NULL; + if (isrc == NULL) + return; + + if (isrc->xi_cookie != NULL) + intr_remove_handler(isrc->xi_cookie); + xen_intr_release_isrc(isrc); +} + +void +xen_intr_signal(xen_intr_handle_t handle) +{ + struct xenisrc *isrc; + + isrc = xen_intr_isrc(handle); + if (isrc != NULL) { + KASSERT(isrc->xi_type == EVTCHN_TYPE_PORT || + isrc->xi_type == EVTCHN_TYPE_IPI, + ("evtchn_signal on something other than a local port")); + struct evtchn_send send = { .port = isrc->xi_port }; + (void)HYPERVISOR_event_channel_op(EVTCHNOP_send, &send); + } +} + +evtchn_port_t +xen_intr_port(xen_intr_handle_t handle) +{ + struct xenisrc *isrc; + + isrc = xen_intr_isrc(handle); + if (isrc == NULL) + return (0); + + return (isrc->xi_port); +} + +int +xen_intr_add_handler(device_t dev, driver_filter_t filter, + driver_intr_t handler, void *arg, enum intr_type flags, + xen_intr_handle_t handle) +{ + struct xenisrc *isrc; + int error; + + isrc = xen_intr_isrc(handle); + if (isrc == NULL || isrc->xi_cookie != NULL) + return (EINVAL); + + error = intr_add_handler(device_get_nameunit(dev), isrc->xi_vector, + filter, handler, arg, flags|INTR_EXCL, &isrc->xi_cookie); + if (error != 0) { + device_printf(dev, + "xen_intr_add_handler: intr_add_handler failed: %d\n", + error); + } + + return (error); +} + +#ifdef DDB +static const char * +xen_intr_print_type(enum evtchn_type type) +{ + static const char *evtchn_type_to_string[EVTCHN_TYPE_COUNT] = { + [EVTCHN_TYPE_UNBOUND] = "UNBOUND", + [EVTCHN_TYPE_PIRQ] = "PIRQ", + [EVTCHN_TYPE_VIRQ] = "VIRQ", + [EVTCHN_TYPE_IPI] = "IPI", + [EVTCHN_TYPE_PORT] = "PORT", + }; + + if (type >= EVTCHN_TYPE_COUNT) + return ("UNKNOWN"); + + return (evtchn_type_to_string[type]); +} + +static void +xen_intr_dump_port(struct xenisrc *isrc) +{ + struct xen_intr_pcpu_data *pcpu; + shared_info_t *s = HYPERVISOR_shared_info; + int i; + + db_printf("Port %d Type: %s\n", + isrc->xi_port, xen_intr_print_type(isrc->xi_type)); + if (isrc->xi_type == EVTCHN_TYPE_PIRQ) { + db_printf("\tPirq: %d ActiveHi: %d EdgeTrigger: %d " + "NeedsEOI: %d Shared: %d\n", + isrc->xi_pirq, isrc->xi_activehi, isrc->xi_edgetrigger, + !!test_bit(isrc->xi_pirq, xen_intr_pirq_eoi_map), + isrc->xi_shared); + } + if (isrc->xi_type == EVTCHN_TYPE_VIRQ) + db_printf("\tVirq: %d\n", isrc->xi_virq); + + db_printf("\tMasked: %d Pending: %d\n", + !!test_bit(isrc->xi_port, &s->evtchn_mask[0]), + !!test_bit(isrc->xi_port, &s->evtchn_pending[0])); + + db_printf("\tPer-CPU Masks: "); + CPU_FOREACH(i) { + pcpu = DPCPU_ID_PTR(i, xen_intr_pcpu); + db_printf("cpu#%d: %d ", i, + !!test_bit(isrc->xi_port, pcpu->evtchn_enabled)); + } + db_printf("\n"); +} + +DB_SHOW_COMMAND(xen_evtchn, db_show_xen_evtchn) +{ + int i; + + if (!xen_domain()) { + db_printf("Only available on Xen guests\n"); + return; + } + + for (i = 0; i < NR_EVENT_CHANNELS; i++) { + struct xenisrc *isrc; + + isrc = xen_intr_port_to_isrc[i]; + if (isrc == NULL) + continue; + + xen_intr_dump_port(isrc); + } +} +#endif /* DDB */ -- 2.1.0