--- /dev/null
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/slab.h>
+#include <linux/dma-direct.h>
+#include <linux/kthread.h>
+
+#include <xen/swiotlb-xen.h>
+#include <xen/pv-iommu-xen.h>
+#include <xen/page.h>
+#include <xen/grant_table.h>
+
+#include <asm/xen/hypervisor.h>
+#include <asm/xen/hypercall.h>
+#include <asm/xen/page.h>
+#include <xen/xen.h>
+
+#include <trace/events/swiotlb.h>
+
+static int xen_pv_iommu_op(struct xen_iommu_op *op)
+{
+ struct xen_iommu_op_buf buf;
+ int rc;
+
+ set_xen_guest_handle(buf.h, op);
+ buf.size = sizeof(*op);
+
+ rc = HYPERVISOR_iommu_op(1, &buf);
+ if (rc)
+ return rc;
+
+ return op->status;
+}
+
+int __init xen_pv_iommu_detect(void)
+{
+ struct xen_iommu_op op = {
+ .op = XEN_IOMMUOP_enable_modification,
+ };
+ int rc;
+
+ rc = xen_pv_iommu_op(&op);
+ if (rc)
+ return 0;
+
+ return 1;
+}
+
+extern int xen_pv_iommu;
+
+static void __xen_pv_iommu_err(const char *func, int line, int err)
+{
+ if (!err)
+ return;
+
+ pr_err("%s: failed at line %d (%d)\n", func, line, err);
+ BUG();
+}
+
+#define xen_pv_iommu_err(_rc) __xen_pv_iommu_err(__func__, __LINE__, (_rc))
+
+void xen_pv_iommu_map(unsigned long pfn)
+{
+ unsigned long mfn = __pfn_to_mfn(pfn);
+ struct xen_iommu_op op = {
+ .op = XEN_IOMMUOP_map,
+ .u.map.domid = DOMID_SELF,
+ .u.map.flags = XEN_IOMMUOP_map_all,
+ .u.map.dfn = pfn,
+ .u.map.u.gfn = mfn,
+ };
+ int rc;
+
+ if (!xen_pv_iommu)
+ return;
+
+ BUG_ON((mfn == INVALID_P2M_ENTRY) ||
+ (mfn & IDENTITY_FRAME_BIT));
+
+ rc = xen_pv_iommu_op(&op);
+ xen_pv_iommu_err(rc);
+}
+
+void xen_pv_iommu_unmap(unsigned long pfn)
+{
+ unsigned long mfn = __pfn_to_mfn(pfn);
+ struct xen_iommu_op op = {
+ .op = XEN_IOMMUOP_unmap,
+ .u.unmap.domid = DOMID_SELF,
+ .u.unmap.flags = XEN_IOMMUOP_unmap_all,
+ .u.unmap.dfn = pfn,
+ .u.unmap.u.gfn = mfn,
+ };
+ int rc;
+
+ if (!xen_pv_iommu)
+ return;
+
+ BUG_ON((mfn == INVALID_P2M_ENTRY) ||
+ (mfn & IDENTITY_FRAME_BIT));
+
+ rc = xen_pv_iommu_op(&op);
+ xen_pv_iommu_err(rc);
+}
+
+static bool build_map_op(struct xen_iommu_op *op,
+ struct xen_iommu_op_buf *buf,
+ unsigned long pfn, bool readonly)
+{
+ unsigned long mfn = __pfn_to_mfn(pfn);
+
+ if ((mfn == INVALID_P2M_ENTRY) ||
+ (mfn & IDENTITY_FRAME_BIT))
+ return false;
+
+ op->op = XEN_IOMMUOP_map;
+ op->u.map.dfn = pfn;
+ op->u.map.flags = XEN_IOMMUOP_map_all;
+
+ if (readonly)
+ op->u.map.flags |= XEN_IOMMUOP_map_readonly;
+
+ if (mfn & FOREIGN_FRAME_BIT) {
+ struct xen_page_foreign *foreign;
+
+ op->u.map.flags |= XEN_IOMMUOP_map_gref;
+
+ foreign = xen_page_foreign(xen_pfn_to_page(pfn));
+ BUG_ON(!foreign);
+
+ op->u.map.domid = foreign->domid;
+ op->u.map.u.gref = foreign->gref;
+ } else {
+ op->u.map.domid = DOMID_SELF;
+ op->u.map.u.gfn = mfn;
+ }
+
+ set_xen_guest_handle(buf->h, op);
+ buf->size = sizeof(*op);
+
+ return true;
+}
+
+#define BATCH_SIZE 32
+
+static void map_pfns(unsigned long pfn, unsigned int nr_frames,
+ bool readonly)
+{
+ struct xen_iommu_op op[BATCH_SIZE];
+ struct xen_iommu_op_buf buf[BATCH_SIZE];
+ unsigned int count;
+ int rc;
+
+ memset(op, 0, sizeof(op));
+ memset(buf, 0, sizeof(buf));
+ count = 0;
+
+ while (nr_frames-- != 0) {
+ if (build_map_op(&op[count], &buf[count], pfn, readonly))
+ count++;
+
+ pfn++;
+
+ if (count < ARRAY_SIZE(buf))
+ continue;
+
+ rc = HYPERVISOR_iommu_op(count, buf);
+ while (!rc && count-- != 0)
+ rc = op[count].status;
+
+ xen_pv_iommu_err(rc);
+
+ memset(op, 0, sizeof(op));
+ memset(buf, 0, sizeof(buf));
+ count = 0;
+ }
+
+ if (count) {
+ rc = HYPERVISOR_iommu_op(count, buf);
+ while (!rc && count-- != 0)
+ rc = op[count].status;
+
+ xen_pv_iommu_err(rc);
+ }
+}
+
+static bool build_unmap_op(struct xen_iommu_op *op,
+ struct xen_iommu_op_buf *buf,
+ unsigned long pfn)
+{
+ unsigned long mfn = __pfn_to_mfn(pfn);
+
+ if ((mfn == INVALID_P2M_ENTRY) ||
+ (mfn & IDENTITY_FRAME_BIT))
+ return false;
+
+ op->op = XEN_IOMMUOP_unmap;
+ op->u.unmap.dfn = pfn;
+ op->u.unmap.flags = XEN_IOMMUOP_unmap_all;
+
+ if (mfn & FOREIGN_FRAME_BIT) {
+ struct xen_page_foreign *foreign;
+
+ op->u.unmap.flags |= XEN_IOMMUOP_unmap_gref;
+
+ foreign = xen_page_foreign(xen_pfn_to_page(pfn));
+ BUG_ON(!foreign);
+
+ op->u.unmap.domid = foreign->domid;
+ op->u.unmap.u.gref = foreign->gref;
+ } else {
+ op->u.unmap.domid = DOMID_SELF;
+ op->u.unmap.u.gfn = mfn;
+ }
+
+ set_xen_guest_handle(buf->h, op);
+ buf->size = sizeof(*op);
+
+ return true;
+}
+
+static void unmap_pfns(unsigned long pfn, unsigned int nr_frames)
+{
+ struct xen_iommu_op op[BATCH_SIZE];
+ struct xen_iommu_op_buf buf[BATCH_SIZE];
+ unsigned int count;
+ int rc;
+
+ memset(op, 0, sizeof(op));
+ memset(buf, 0, sizeof(buf));
+ count = 0;
+
+ while (nr_frames-- != 0) {
+ if (build_unmap_op(&op[count], &buf[count], pfn))
+ count++;
+
+ pfn++;
+
+ if (count < ARRAY_SIZE(buf))
+ continue;
+
+ rc = HYPERVISOR_iommu_op(count, buf);
+ while (!rc && count-- != 0)
+ rc = op[count].status;
+
+ xen_pv_iommu_err(rc);
+
+ memset(op, 0, sizeof(op));
+ memset(buf, 0, sizeof(buf));
+ count = 0;
+ }
+
+ if (count) {
+ rc = HYPERVISOR_iommu_op(count, buf);
+ while (!rc && count-- != 0)
+ rc = op[count].status;
+
+ xen_pv_iommu_err(rc);
+ }
+}
+
+static void setup_dom0_map(void)
+{
+ pr_info("%s: ====>\n", __func__);
+
+ map_pfns(0, max_pfn, false);
+
+ pr_info("%s: <====\n", __func__);
+}
+
+void __init xen_pv_iommu_early_init(void)
+{
+ pr_info("%s: ====>\n", __func__);
+
+ setup_dom0_map();
+
+ swiotlb_init(1);
+
+ pr_info("%s: <====\n", __func__);
+}
+
+void __init xen_pv_iommu_late_init(void)
+{
+ pr_info("%s: <===>\n", __func__);
+}
+
+static inline int range_is_foreign(phys_addr_t paddr, size_t size)
+{
+ phys_addr_t end = paddr + size - 1;
+
+ paddr &= XEN_PAGE_MASK;
+
+ while (paddr <= end) {
+ unsigned long mfn = __pfn_to_mfn(XEN_PFN_DOWN(paddr));
+
+ if (mfn & FOREIGN_FRAME_BIT)
+ return 1;
+
+ paddr += XEN_PAGE_SIZE;
+ }
+
+ return 0;
+}
+
+static unsigned int nr_frames(phys_addr_t paddr, size_t size)
+{
+ phys_addr_t end = paddr + size - 1;
+ unsigned int count = 0;
+
+ paddr &= XEN_PAGE_MASK;
+
+ while (paddr <= end) {
+ count++;
+ paddr += XEN_PAGE_SIZE;
+ }
+
+ return count;
+}
+
+#define XEN_PV_IOMMU_ERROR_CODE (~(dma_addr_t)0x0)
+
+static dma_addr_t __map_single(struct device *hwdev, phys_addr_t paddr,
+ size_t size, enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ dma_addr_t dev_addr = paddr;
+ phys_addr_t map;
+
+ BUG_ON(dir == DMA_NONE);
+
+ if (dma_capable(hwdev, dev_addr, size)) {
+ if (range_is_foreign(paddr, size)) {
+ unsigned int count = nr_frames(paddr, size);
+
+ map_pfns(XEN_PFN_DOWN(paddr), count,
+ (dir == DMA_TO_DEVICE));
+ }
+
+ return dev_addr;
+ }
+
+ /*
+ * Oh well, have to allocate and map a bounce buffer.
+ */
+ trace_swiotlb_bounced(hwdev, dev_addr, size, 1);
+
+ map = swiotlb_map_single(hwdev, paddr, size, dir, attrs);
+ if (map == SWIOTLB_MAP_ERROR)
+ return XEN_PV_IOMMU_ERROR_CODE;
+
+ dev_addr = map;
+
+ /*
+ * Ensure that the address returned is DMA'ble
+ */
+ if (dma_capable(hwdev, dev_addr, size))
+ return dev_addr;
+
+ attrs |= DMA_ATTR_SKIP_CPU_SYNC;
+ swiotlb_unmap_single(hwdev, map, size, dir, attrs);
+
+ return XEN_PV_IOMMU_ERROR_CODE;
+}
+
+static void __unmap_single(struct device *hwdev, dma_addr_t dev_addr,
+ size_t size, enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ phys_addr_t paddr = dev_addr;
+
+ if (is_swiotlb_buffer(paddr)) {
+ swiotlb_unmap_single(hwdev, paddr, size, dir, attrs);
+ return;
+ }
+
+ if (range_is_foreign(paddr, size)) {
+ unsigned int count = nr_frames(paddr, size);
+
+ unmap_pfns(XEN_PFN_DOWN(paddr), count);
+ }
+}
+
+static int
+xen_pv_iommu_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
+ int nelems, enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ struct scatterlist *sg;
+ int i;
+
+ for_each_sg(sgl, sg, nelems, i) {
+ phys_addr_t paddr = sg_phys(sg);
+ size_t size = sg->length;
+
+ sg->dma_address = __map_single(hwdev, paddr, size, dir,
+ attrs);
+ sg_dma_len(sg) = size;
+ }
+
+ return nelems;
+}
+
+static void
+xen_pv_iommu_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
+ int nelems, enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ struct scatterlist *sg;
+ int i;
+
+ for_each_sg(sgl, sg, nelems, i) {
+ dma_addr_t dev_addr = sg->dma_address;
+ size_t size = sg_dma_len(sg);
+
+ __unmap_single(hwdev, dev_addr, size, dir, attrs);
+ }
+}
+
+static dma_addr_t xen_pv_iommu_map_page(struct device *hwdev,
+ struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ phys_addr_t paddr = page_to_phys(page) + offset;
+
+ return __map_single(hwdev, paddr, size, dir, attrs);
+}
+
+static void xen_pv_iommu_unmap_page(struct device *hwdev,
+ dma_addr_t dev_addr,
+ size_t size,
+ enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ __unmap_single(hwdev, dev_addr, size, dir, attrs);
+}
+
+static int xen_pv_iommu_mapping_error(struct device *hwdev,
+ dma_addr_t dev_addr)
+{
+ return dev_addr == XEN_PV_IOMMU_ERROR_CODE;
+}
+
+const struct dma_map_ops xen_pv_iommu_dma_ops = {
+ .alloc = swiotlb_alloc,
+ .free = swiotlb_free,
+ .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
+ .sync_single_for_device = swiotlb_sync_single_for_device,
+ .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
+ .sync_sg_for_device = swiotlb_sync_sg_for_device,
+ .map_sg = xen_pv_iommu_map_sg_attrs,
+ .unmap_sg = xen_pv_iommu_unmap_sg_attrs,
+ .map_page = xen_pv_iommu_map_page,
+ .unmap_page = xen_pv_iommu_unmap_page,
+ .dma_supported = dma_direct_supported,
+ .mapping_error = xen_pv_iommu_mapping_error,
+};
projects / people / pauldu / linux.git / commitdiff