From: Paul Durrant Date: Fri, 12 Oct 2018 14:33:25 +0000 (+0100) Subject: stuff X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=refs%2Fheads%2Fpv-iommu13;p=people%2Fpauldu%2Flinux.git stuff Signed-off-by: Paul Durrant --- diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h index e8aed1cd507a..058cf74a36e2 100644 --- a/arch/x86/include/asm/xen/hypercall.h +++ b/arch/x86/include/asm/xen/hypercall.h @@ -52,6 +52,7 @@ #include #include #include +#include struct xen_dm_op_buf; diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c index 37c6056a7bba..df4b11cc8ee3 100644 --- a/arch/x86/xen/pci-swiotlb-xen.c +++ b/arch/x86/xen/pci-swiotlb-xen.c @@ -3,8 +3,11 @@ #include #include #include +#include #include +#include +#include #include #include @@ -17,6 +20,41 @@ #include int xen_swiotlb __read_mostly; +int xen_pv_iommu __read_mostly; + +int __init pci_xen_pv_iommu_detect(void) +{ + if (!xen_pv_domain() || !xen_initial_domain()) + return 0; + + xen_pv_iommu = xen_pv_iommu_detect(); + + return xen_pv_iommu; +} + +void __init pci_xen_pv_iommu_init(void) +{ + if (xen_pv_iommu) { + xen_pv_iommu_early_init(); + dma_ops = &xen_pv_iommu_dma_ops; + +#ifdef CONFIG_PCI + /* Make sure ACS will be enabled */ + pci_request_acs(); +#endif + } +} + +void __init pci_xen_pv_iommu_late_init(void) +{ + if (xen_pv_iommu) + xen_pv_iommu_late_init(); +} + +IOMMU_INIT_FINISH(pci_xen_pv_iommu_detect, + NULL, + pci_xen_pv_iommu_init, + pci_xen_pv_iommu_late_init); /* * pci_xen_swiotlb_detect - set xen_swiotlb to 1 if necessary @@ -89,6 +127,6 @@ int pci_xen_swiotlb_init_late(void) EXPORT_SYMBOL_GPL(pci_xen_swiotlb_init_late); IOMMU_INIT_FINISH(pci_xen_swiotlb_detect, - NULL, + pci_xen_pv_iommu_detect, pci_xen_swiotlb_init, NULL); diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile index 3e542f60f29f..a152da801e60 100644 --- a/drivers/xen/Makefile +++ b/drivers/xen/Makefile @@ -26,7 +26,7 @@ obj-$(CONFIG_XENFS) += xenfs/ obj-$(CONFIG_XEN_SYS_HYPERVISOR) += sys-hypervisor.o obj-$(CONFIG_XEN_PVHVM) += platform-pci.o obj-$(CONFIG_XEN_TMEM) += tmem.o -obj-$(CONFIG_SWIOTLB_XEN) += swiotlb-xen.o +obj-$(CONFIG_SWIOTLB_XEN) += swiotlb-xen.o pv-iommu-xen.o obj-$(CONFIG_XEN_MCE_LOG) += mcelog.o obj-$(CONFIG_XEN_PCIDEV_BACKEND) += xen-pciback/ obj-$(CONFIG_XEN_PRIVCMD) += xen-privcmd.o diff --git a/drivers/xen/biomerge.c b/drivers/xen/biomerge.c index 55ed80c3a17c..c74ab58bb8e8 100644 --- a/drivers/xen/biomerge.c +++ b/drivers/xen/biomerge.c @@ -7,17 +7,10 @@ bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, const struct bio_vec *vec2) { -#if XEN_PAGE_SIZE == PAGE_SIZE - unsigned long bfn1 = pfn_to_bfn(page_to_pfn(vec1->bv_page)); - unsigned long bfn2 = pfn_to_bfn(page_to_pfn(vec2->bv_page)); - - return bfn1 + PFN_DOWN(vec1->bv_offset + vec1->bv_len) == bfn2; -#else /* * XXX: Add support for merging bio_vec when using different page * size in Xen and Linux. */ return false; -#endif } EXPORT_SYMBOL(xen_biovec_phys_mergeable); diff --git a/drivers/xen/mem-reservation.c b/drivers/xen/mem-reservation.c index 3782cf070338..85427a4459ed 100644 --- a/drivers/xen/mem-reservation.c +++ b/drivers/xen/mem-reservation.c @@ -75,6 +75,8 @@ void __xenmem_reservation_va_mapping_reset(unsigned long count, */ BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE); + xen_pv_iommu_unmap(pfn); + if (!PageHighMem(page)) { int ret; @@ -83,7 +85,6 @@ void __xenmem_reservation_va_mapping_reset(unsigned long count, __pte_ma(0), 0); BUG_ON(ret); } - __set_phys_to_machine(pfn, INVALID_P2M_ENTRY); } } EXPORT_SYMBOL_GPL(__xenmem_reservation_va_mapping_reset); diff --git a/drivers/xen/pv-iommu-xen.c b/drivers/xen/pv-iommu-xen.c new file mode 100644 index 000000000000..32a4ed5e4ea8 --- /dev/null +++ b/drivers/xen/pv-iommu-xen.c @@ -0,0 +1,458 @@ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +static int xen_pv_iommu_op(struct xen_iommu_op *op) +{ + struct xen_iommu_op_buf buf; + int rc; + + set_xen_guest_handle(buf.h, op); + buf.size = sizeof(*op); + + rc = HYPERVISOR_iommu_op(1, &buf); + if (rc) + return rc; + + return op->status; +} + +int __init xen_pv_iommu_detect(void) +{ + struct xen_iommu_op op = { + .op = XEN_IOMMUOP_enable_modification, + }; + int rc; + + rc = xen_pv_iommu_op(&op); + if (rc) + return 0; + + return 1; +} + +extern int xen_pv_iommu; + +static void __xen_pv_iommu_err(const char *func, int line, int err) +{ + if (!err) + return; + + pr_err("%s: failed at line %d (%d)\n", func, line, err); + BUG(); +} + +#define xen_pv_iommu_err(_rc) __xen_pv_iommu_err(__func__, __LINE__, (_rc)) + +void xen_pv_iommu_map(unsigned long pfn) +{ + unsigned long mfn = __pfn_to_mfn(pfn); + struct xen_iommu_op op = { + .op = XEN_IOMMUOP_map, + .u.map.domid = DOMID_SELF, + .u.map.flags = XEN_IOMMUOP_map_all, + .u.map.dfn = pfn, + .u.map.u.gfn = mfn, + }; + int rc; + + if (!xen_pv_iommu) + return; + + BUG_ON((mfn == INVALID_P2M_ENTRY) || + (mfn & IDENTITY_FRAME_BIT)); + + rc = xen_pv_iommu_op(&op); + xen_pv_iommu_err(rc); +} + +void xen_pv_iommu_unmap(unsigned long pfn) +{ + unsigned long mfn = __pfn_to_mfn(pfn); + struct xen_iommu_op op = { + .op = XEN_IOMMUOP_unmap, + .u.unmap.domid = DOMID_SELF, + .u.unmap.flags = XEN_IOMMUOP_unmap_all, + .u.unmap.dfn = pfn, + .u.unmap.u.gfn = mfn, + }; + int rc; + + if (!xen_pv_iommu) + return; + + BUG_ON((mfn == INVALID_P2M_ENTRY) || + (mfn & IDENTITY_FRAME_BIT)); + + rc = xen_pv_iommu_op(&op); + xen_pv_iommu_err(rc); +} + +static bool build_map_op(struct xen_iommu_op *op, + struct xen_iommu_op_buf *buf, + unsigned long pfn, bool readonly) +{ + unsigned long mfn = __pfn_to_mfn(pfn); + + if ((mfn == INVALID_P2M_ENTRY) || + (mfn & IDENTITY_FRAME_BIT)) + return false; + + op->op = XEN_IOMMUOP_map; + op->u.map.dfn = pfn; + op->u.map.flags = XEN_IOMMUOP_map_all; + + if (readonly) + op->u.map.flags |= XEN_IOMMUOP_map_readonly; + + if (mfn & FOREIGN_FRAME_BIT) { + struct xen_page_foreign *foreign; + + op->u.map.flags |= XEN_IOMMUOP_map_gref; + + foreign = xen_page_foreign(xen_pfn_to_page(pfn)); + BUG_ON(!foreign); + + op->u.map.domid = foreign->domid; + op->u.map.u.gref = foreign->gref; + } else { + op->u.map.domid = DOMID_SELF; + op->u.map.u.gfn = mfn; + } + + set_xen_guest_handle(buf->h, op); + buf->size = sizeof(*op); + + return true; +} + +#define BATCH_SIZE 32 + +static void map_pfns(unsigned long pfn, unsigned int nr_frames, + bool readonly) +{ + struct xen_iommu_op op[BATCH_SIZE]; + struct xen_iommu_op_buf buf[BATCH_SIZE]; + unsigned int count; + int rc; + + memset(op, 0, sizeof(op)); + memset(buf, 0, sizeof(buf)); + count = 0; + + while (nr_frames-- != 0) { + if (build_map_op(&op[count], &buf[count], pfn, readonly)) + count++; + + pfn++; + + if (count < ARRAY_SIZE(buf)) + continue; + + rc = HYPERVISOR_iommu_op(count, buf); + while (!rc && count-- != 0) + rc = op[count].status; + + xen_pv_iommu_err(rc); + + memset(op, 0, sizeof(op)); + memset(buf, 0, sizeof(buf)); + count = 0; + } + + if (count) { + rc = HYPERVISOR_iommu_op(count, buf); + while (!rc && count-- != 0) + rc = op[count].status; + + xen_pv_iommu_err(rc); + } +} + +static bool build_unmap_op(struct xen_iommu_op *op, + struct xen_iommu_op_buf *buf, + unsigned long pfn) +{ + unsigned long mfn = __pfn_to_mfn(pfn); + + if ((mfn == INVALID_P2M_ENTRY) || + (mfn & IDENTITY_FRAME_BIT)) + return false; + + op->op = XEN_IOMMUOP_unmap; + op->u.unmap.dfn = pfn; + op->u.unmap.flags = XEN_IOMMUOP_unmap_all; + + if (mfn & FOREIGN_FRAME_BIT) { + struct xen_page_foreign *foreign; + + op->u.unmap.flags |= XEN_IOMMUOP_unmap_gref; + + foreign = xen_page_foreign(xen_pfn_to_page(pfn)); + BUG_ON(!foreign); + + op->u.unmap.domid = foreign->domid; + op->u.unmap.u.gref = foreign->gref; + } else { + op->u.unmap.domid = DOMID_SELF; + op->u.unmap.u.gfn = mfn; + } + + set_xen_guest_handle(buf->h, op); + buf->size = sizeof(*op); + + return true; +} + +static void unmap_pfns(unsigned long pfn, unsigned int nr_frames) +{ + struct xen_iommu_op op[BATCH_SIZE]; + struct xen_iommu_op_buf buf[BATCH_SIZE]; + unsigned int count; + int rc; + + memset(op, 0, sizeof(op)); + memset(buf, 0, sizeof(buf)); + count = 0; + + while (nr_frames-- != 0) { + if (build_unmap_op(&op[count], &buf[count], pfn)) + count++; + + pfn++; + + if (count < ARRAY_SIZE(buf)) + continue; + + rc = HYPERVISOR_iommu_op(count, buf); + while (!rc && count-- != 0) + rc = op[count].status; + + xen_pv_iommu_err(rc); + + memset(op, 0, sizeof(op)); + memset(buf, 0, sizeof(buf)); + count = 0; + } + + if (count) { + rc = HYPERVISOR_iommu_op(count, buf); + while (!rc && count-- != 0) + rc = op[count].status; + + xen_pv_iommu_err(rc); + } +} + +static void setup_dom0_map(void) +{ + pr_info("%s: ====>\n", __func__); + + map_pfns(0, max_pfn, false); + + pr_info("%s: <====\n", __func__); +} + +void __init xen_pv_iommu_early_init(void) +{ + pr_info("%s: ====>\n", __func__); + + setup_dom0_map(); + + swiotlb_init(1); + + pr_info("%s: <====\n", __func__); +} + +void __init xen_pv_iommu_late_init(void) +{ + pr_info("%s: <===>\n", __func__); +} + +static inline int range_is_foreign(phys_addr_t paddr, size_t size) +{ + phys_addr_t end = paddr + size - 1; + + paddr &= XEN_PAGE_MASK; + + while (paddr <= end) { + unsigned long mfn = __pfn_to_mfn(XEN_PFN_DOWN(paddr)); + + if (mfn & FOREIGN_FRAME_BIT) + return 1; + + paddr += XEN_PAGE_SIZE; + } + + return 0; +} + +static unsigned int nr_frames(phys_addr_t paddr, size_t size) +{ + phys_addr_t end = paddr + size - 1; + unsigned int count = 0; + + paddr &= XEN_PAGE_MASK; + + while (paddr <= end) { + count++; + paddr += XEN_PAGE_SIZE; + } + + return count; +} + +#define XEN_PV_IOMMU_ERROR_CODE (~(dma_addr_t)0x0) + +static dma_addr_t __map_single(struct device *hwdev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + dma_addr_t dev_addr = paddr; + phys_addr_t map; + + BUG_ON(dir == DMA_NONE); + + if (dma_capable(hwdev, dev_addr, size)) { + if (range_is_foreign(paddr, size)) { + unsigned int count = nr_frames(paddr, size); + + map_pfns(XEN_PFN_DOWN(paddr), count, + (dir == DMA_TO_DEVICE)); + } + + return dev_addr; + } + + /* + * Oh well, have to allocate and map a bounce buffer. + */ + trace_swiotlb_bounced(hwdev, dev_addr, size, 1); + + map = swiotlb_map_single(hwdev, paddr, size, dir, attrs); + if (map == SWIOTLB_MAP_ERROR) + return XEN_PV_IOMMU_ERROR_CODE; + + dev_addr = map; + + /* + * Ensure that the address returned is DMA'ble + */ + if (dma_capable(hwdev, dev_addr, size)) + return dev_addr; + + attrs |= DMA_ATTR_SKIP_CPU_SYNC; + swiotlb_unmap_single(hwdev, map, size, dir, attrs); + + return XEN_PV_IOMMU_ERROR_CODE; +} + +static void __unmap_single(struct device *hwdev, dma_addr_t dev_addr, + size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + phys_addr_t paddr = dev_addr; + + if (is_swiotlb_buffer(paddr)) { + swiotlb_unmap_single(hwdev, paddr, size, dir, attrs); + return; + } + + if (range_is_foreign(paddr, size)) { + unsigned int count = nr_frames(paddr, size); + + unmap_pfns(XEN_PFN_DOWN(paddr), count); + } +} + +static int +xen_pv_iommu_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, + int nelems, enum dma_data_direction dir, + unsigned long attrs) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nelems, i) { + phys_addr_t paddr = sg_phys(sg); + size_t size = sg->length; + + sg->dma_address = __map_single(hwdev, paddr, size, dir, + attrs); + sg_dma_len(sg) = size; + } + + return nelems; +} + +static void +xen_pv_iommu_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl, + int nelems, enum dma_data_direction dir, + unsigned long attrs) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nelems, i) { + dma_addr_t dev_addr = sg->dma_address; + size_t size = sg_dma_len(sg); + + __unmap_single(hwdev, dev_addr, size, dir, attrs); + } +} + +static dma_addr_t xen_pv_iommu_map_page(struct device *hwdev, + struct page *page, + unsigned long offset, size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + phys_addr_t paddr = page_to_phys(page) + offset; + + return __map_single(hwdev, paddr, size, dir, attrs); +} + +static void xen_pv_iommu_unmap_page(struct device *hwdev, + dma_addr_t dev_addr, + size_t size, + enum dma_data_direction dir, + unsigned long attrs) +{ + __unmap_single(hwdev, dev_addr, size, dir, attrs); +} + +static int xen_pv_iommu_mapping_error(struct device *hwdev, + dma_addr_t dev_addr) +{ + return dev_addr == XEN_PV_IOMMU_ERROR_CODE; +} + +const struct dma_map_ops xen_pv_iommu_dma_ops = { + .alloc = swiotlb_alloc, + .free = swiotlb_free, + .sync_single_for_cpu = swiotlb_sync_single_for_cpu, + .sync_single_for_device = swiotlb_sync_single_for_device, + .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, + .sync_sg_for_device = swiotlb_sync_sg_for_device, + .map_sg = xen_pv_iommu_map_sg_attrs, + .unmap_sg = xen_pv_iommu_unmap_sg_attrs, + .map_page = xen_pv_iommu_map_page, + .unmap_page = xen_pv_iommu_unmap_page, + .dma_supported = dma_direct_supported, + .mapping_error = xen_pv_iommu_mapping_error, +}; diff --git a/include/xen/pv-iommu-xen.h b/include/xen/pv-iommu-xen.h new file mode 100644 index 000000000000..48c16b1561e6 --- /dev/null +++ b/include/xen/pv-iommu-xen.h @@ -0,0 +1,14 @@ +#ifndef __LINUX_PV_IOMMU_XEN_H +#define __LINUX_PV_IOMMU_XEN_H + +#include + +extern int xen_pv_iommu_detect(void); +extern void xen_pv_iommu_early_init(void); +extern void xen_pv_iommu_late_init(void); +extern const struct dma_map_ops xen_pv_iommu_dma_ops; + +extern void xen_pv_iommu_map(unsigned long pfn); +extern void xen_pv_iommu_unmap(unsigned long pfn); + +#endif /* __LINUX_PV_IOMMU_XEN_H */