From 3fdb2ba3063c84bf1adfd80a068d89f27bbaaf86 Mon Sep 17 00:00:00 2001 From: Keir Fraser Date: Thu, 19 Mar 2009 10:20:11 +0000 Subject: [PATCH] Xen: use proper device ID to search VT-d unit for ARI and SR-IOV device PCIe Alternative Routing-ID Interpretation (ARI) ECN defines the Extended Function -- a function whose function number is greater than 7 within an ARI Device. Intel VT-d spec 1.2 section 8.3.2 specifies that the Extended Function is under the scope of the same remapping unit as the traditional function. The hypervisor needs to know if a function is Extended Function so it can find proper DMAR for it. And section 8.3.3 specifies that the SR-IOV Virtual Function is under the scope of the same remapping unit as the Physical Function. The hypervisor also needs to know if a function is the Virtual Function and which Physical Function it's associated with for same reason. Signed-off-by: Yu Zhao --- xen/arch/ia64/xen/hypercall.c | 22 ++++++++++++++ xen/arch/x86/physdev.c | 26 ++++++++++++++++ xen/drivers/passthrough/pci.c | 41 ++++++++++++++++++++++++++ xen/drivers/passthrough/vtd/dmar.c | 14 ++++++++- xen/drivers/passthrough/vtd/dmar.h | 2 +- xen/drivers/passthrough/vtd/intremap.c | 4 +-- xen/drivers/passthrough/vtd/iommu.c | 14 ++++++--- xen/include/public/physdev.h | 16 ++++++++++ xen/include/xen/pci.h | 11 +++++++ 9 files changed, 142 insertions(+), 8 deletions(-) diff --git a/xen/arch/ia64/xen/hypercall.c b/xen/arch/ia64/xen/hypercall.c index a5828b1816..884bf7b4c3 100644 --- a/xen/arch/ia64/xen/hypercall.c +++ b/xen/arch/ia64/xen/hypercall.c @@ -674,6 +674,28 @@ long do_physdev_op(int cmd, XEN_GUEST_HANDLE(void) arg) break; } + case PHYSDEVOP_manage_pci_add_ext: { + struct physdev_manage_pci_ext manage_pci_ext; + struct pci_dev_info pdev_info; + + ret = -EPERM; + if ( !IS_PRIV(current->domain) ) + break; + + ret = -EFAULT; + if ( copy_from_guest(&manage_pci_ext, arg, 1) != 0 ) + break; + + pdev_info.is_extfn = manage_pci_ext.is_extfn; + pdev_info.is_virtfn = manage_pci_ext.is_virtfn; + pdev_info.physfn.bus = manage_pci_ext.physfn.bus; + pdev_info.physfn.devfn = manage_pci_ext.physfn.devfn; + ret = pci_add_device_ext(manage_pci_ext.bus, + manage_pci_ext.devfn, + &pdev_info); + break; + } + default: ret = -ENOSYS; printk("not implemented do_physdev_op: %d\n", cmd); diff --git a/xen/arch/x86/physdev.c b/xen/arch/x86/physdev.c index 9c55237a01..52633a6ed6 100644 --- a/xen/arch/x86/physdev.c +++ b/xen/arch/x86/physdev.c @@ -421,6 +421,32 @@ ret_t do_physdev_op(int cmd, XEN_GUEST_HANDLE(void) arg) break; } + case PHYSDEVOP_manage_pci_add_ext: { + struct physdev_manage_pci_ext manage_pci_ext; + struct pci_dev_info pdev_info; + + ret = -EPERM; + if ( !IS_PRIV(current->domain) ) + break; + + ret = -EFAULT; + if ( copy_from_guest(&manage_pci_ext, arg, 1) != 0 ) + break; + + ret = -EINVAL; + if ( (manage_pci_ext.is_extfn > 1) || (manage_pci_ext.is_virtfn > 1) ) + break; + + pdev_info.is_extfn = manage_pci_ext.is_extfn; + pdev_info.is_virtfn = manage_pci_ext.is_virtfn; + pdev_info.physfn.bus = manage_pci_ext.physfn.bus; + pdev_info.physfn.devfn = manage_pci_ext.physfn.devfn; + ret = pci_add_device_ext(manage_pci_ext.bus, + manage_pci_ext.devfn, + &pdev_info); + break; + } + case PHYSDEVOP_restore_msi: { struct physdev_restore_msi restore_msi; struct pci_dev *pdev; diff --git a/xen/drivers/passthrough/pci.c b/xen/drivers/passthrough/pci.c index 94dfb4e53d..d85a4675a0 100644 --- a/xen/drivers/passthrough/pci.c +++ b/xen/drivers/passthrough/pci.c @@ -143,6 +143,47 @@ int pci_remove_device(u8 bus, u8 devfn) return ret; } +int pci_add_device_ext(u8 bus, u8 devfn, struct pci_dev_info *info) +{ + int ret; + char *pdev_type; + struct pci_dev *pdev; + + if (info->is_extfn) + pdev_type = "Extended Function"; + else if (info->is_virtfn) + pdev_type = "Virtual Function"; + else + return -EINVAL;; + + + ret = -ENOMEM; + spin_lock(&pcidevs_lock); + pdev = alloc_pdev(bus, devfn); + if ( !pdev ) + goto out; + + pdev->info = *info; + + ret = 0; + if ( !pdev->domain ) + { + pdev->domain = dom0; + ret = iommu_add_device(pdev); + if ( ret ) + goto out; + + list_add(&pdev->domain_list, &dom0->arch.pdev_list); + } + +out: + spin_unlock(&pcidevs_lock); + printk(XENLOG_DEBUG "PCI add %s %02x:%02x.%x\n", pdev_type, + bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + + return ret; +} + static void pci_clean_dpci_irqs(struct domain *d) { struct hvm_irq_dpci *hvm_irq_dpci = NULL; diff --git a/xen/drivers/passthrough/vtd/dmar.c b/xen/drivers/passthrough/vtd/dmar.c index ff32f1e9cf..f5d7b786ff 100644 --- a/xen/drivers/passthrough/vtd/dmar.c +++ b/xen/drivers/passthrough/vtd/dmar.c @@ -152,12 +152,24 @@ static int __init acpi_register_atsr_unit(struct acpi_atsr_unit *atsr) return 0; } -struct acpi_drhd_unit * acpi_find_matched_drhd_unit(u8 bus, u8 devfn) +struct acpi_drhd_unit * acpi_find_matched_drhd_unit(struct pci_dev *pdev) { + u8 bus, devfn; struct acpi_drhd_unit *drhd; struct acpi_drhd_unit *found = NULL, *include_all = NULL; int i; + if (pdev->info.is_extfn) { + bus = pdev->bus; + devfn = 0; + } else if (pdev->info.is_virtfn) { + bus = pdev->info.physfn.bus; + devfn = PCI_SLOT(pdev->info.physfn.devfn) ? 0 : pdev->info.physfn.devfn; + } else { + bus = pdev->bus; + devfn = pdev->devfn; + } + list_for_each_entry ( drhd, &acpi_drhd_units, list ) { for (i = 0; i < drhd->scope.devices_cnt; i++) diff --git a/xen/drivers/passthrough/vtd/dmar.h b/xen/drivers/passthrough/vtd/dmar.h index c2132e2a0f..3664b19d21 100644 --- a/xen/drivers/passthrough/vtd/dmar.h +++ b/xen/drivers/passthrough/vtd/dmar.h @@ -79,7 +79,7 @@ struct acpi_atsr_unit { for (idx = 0; (bdf = rmrr->scope.devices[idx]) && \ idx < rmrr->scope.devices_cnt; idx++) -struct acpi_drhd_unit * acpi_find_matched_drhd_unit(u8 bus, u8 devfn); +struct acpi_drhd_unit * acpi_find_matched_drhd_unit(struct pci_dev *pdev); struct acpi_atsr_unit * acpi_find_matched_atsr_unit(u8 bus, u8 devfn); void dmar_scope_add_buses(struct dmar_scope *scope, u16 sec, u16 sub); void dmar_scope_remove_buses(struct dmar_scope *scope, u16 sec, u16 sub); diff --git a/xen/drivers/passthrough/vtd/intremap.c b/xen/drivers/passthrough/vtd/intremap.c index 063031799d..c0d67bf25d 100644 --- a/xen/drivers/passthrough/vtd/intremap.c +++ b/xen/drivers/passthrough/vtd/intremap.c @@ -450,7 +450,7 @@ void msi_msg_read_remap_rte( struct iommu *iommu = NULL; struct ir_ctrl *ir_ctrl; - drhd = acpi_find_matched_drhd_unit(pdev->bus, pdev->devfn); + drhd = acpi_find_matched_drhd_unit(pdev); iommu = drhd->iommu; ir_ctrl = iommu_ir_ctrl(iommu); @@ -468,7 +468,7 @@ void msi_msg_write_remap_rte( struct iommu *iommu = NULL; struct ir_ctrl *ir_ctrl; - drhd = acpi_find_matched_drhd_unit(pdev->bus, pdev->devfn); + drhd = acpi_find_matched_drhd_unit(pdev); iommu = drhd->iommu; ir_ctrl = iommu_ir_ctrl(iommu); diff --git a/xen/drivers/passthrough/vtd/iommu.c b/xen/drivers/passthrough/vtd/iommu.c index 9d6583757f..af58d7dc4d 100644 --- a/xen/drivers/passthrough/vtd/iommu.c +++ b/xen/drivers/passthrough/vtd/iommu.c @@ -1193,8 +1193,11 @@ static int domain_context_mapping(struct domain *domain, u8 bus, u8 devfn) u16 sec_bus, sub_bus; u32 type; u8 secbus, secdevfn; + struct pci_dev *pdev = pci_get_pdev(bus, devfn); - drhd = acpi_find_matched_drhd_unit(bus, devfn); + BUG_ON(!pdev); + + drhd = acpi_find_matched_drhd_unit(pdev); if ( !drhd ) return -ENODEV; @@ -1319,8 +1322,11 @@ static int domain_context_unmap(struct domain *domain, u8 bus, u8 devfn) int ret = 0; u32 type; u8 secbus, secdevfn; + struct pci_dev *pdev = pci_get_pdev(bus, devfn); + + BUG_ON(!pdev); - drhd = acpi_find_matched_drhd_unit(bus, devfn); + drhd = acpi_find_matched_drhd_unit(pdev); if ( !drhd ) return -ENODEV; @@ -1392,7 +1398,7 @@ static int reassign_device_ownership( if (!pdev) return -ENODEV; - drhd = acpi_find_matched_drhd_unit(bus, devfn); + drhd = acpi_find_matched_drhd_unit(pdev); pdev_iommu = drhd->iommu; domain_context_unmap(source, bus, devfn); @@ -1405,7 +1411,7 @@ static int reassign_device_ownership( for_each_pdev ( source, pdev ) { - drhd = acpi_find_matched_drhd_unit(pdev->bus, pdev->devfn); + drhd = acpi_find_matched_drhd_unit(pdev); if ( drhd->iommu == pdev_iommu ) { found = 1; diff --git a/xen/include/public/physdev.h b/xen/include/public/physdev.h index 72a42cf265..41f3c37649 100644 --- a/xen/include/public/physdev.h +++ b/xen/include/public/physdev.h @@ -183,6 +183,22 @@ struct physdev_manage_pci { typedef struct physdev_manage_pci physdev_manage_pci_t; DEFINE_XEN_GUEST_HANDLE(physdev_manage_pci_t); +#define PHYSDEVOP_manage_pci_add_ext 20 +struct physdev_manage_pci_ext { + /* IN */ + uint8_t bus; + uint8_t devfn; + unsigned is_extfn; + unsigned is_virtfn; + struct { + uint8_t bus; + uint8_t devfn; + } physfn; +}; + +typedef struct physdev_manage_pci_ext physdev_manage_pci_ext_t; +DEFINE_XEN_GUEST_HANDLE(physdev_manage_pci_ext_t); + #define PHYSDEVOP_restore_msi 19 struct physdev_restore_msi { /* IN */ diff --git a/xen/include/xen/pci.h b/xen/include/xen/pci.h index 34f79b7a81..6827e0d677 100644 --- a/xen/include/xen/pci.h +++ b/xen/include/xen/pci.h @@ -31,6 +31,15 @@ #define MAX_MSIX_TABLE_ENTRIES 2048 #define MAX_MSIX_TABLE_PAGES 8 +struct pci_dev_info { + unsigned is_extfn; + unsigned is_virtfn; + struct { + u8 bus; + u8 devfn; + } physfn; +}; + struct pci_dev { struct list_head alldevs_list; struct list_head domain_list; @@ -43,6 +52,7 @@ struct pci_dev { struct domain *domain; const u8 bus; const u8 devfn; + struct pci_dev_info info; }; #define for_each_pdev(domain, pdev) \ @@ -64,6 +74,7 @@ struct pci_dev *pci_lock_domain_pdev(struct domain *d, int bus, int devfn); void pci_release_devices(struct domain *d); int pci_add_device(u8 bus, u8 devfn); int pci_remove_device(u8 bus, u8 devfn); +int pci_add_device_ext(u8 bus, u8 devfn, struct pci_dev_info *info); struct pci_dev *pci_get_pdev(int bus, int devfn); struct pci_dev *pci_get_pdev_by_domain(struct domain *d, int bus, int devfn); -- 2.39.5