#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/string.h>
-#include <linux/swiotlb.h>
+#include <xen/swiotlb.h>
#include <linux/pfn.h>
#include <linux/types.h>
#include <linux/ctype.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/iommu-helper.h>
+#include <asm/iommu.h>
+
+#include <xen/interface/xen.h>
+#include <xen/grant_table.h>
+#include <xen/page.h>
+#include <xen/xen-ops.h>
#define OFFSET(val,align) ((unsigned long) \
( (val) & ( (align) - 1)))
*/
static DEFINE_SPINLOCK(xen_io_tlb_lock);
+
void * __weak __init xen_swiotlb_alloc_boot(size_t size, unsigned long nslabs)
{
return alloc_bootmem_low_pages(size);
dma_addr_t __weak xen_swiotlb_phys_to_bus(struct device *hwdev, phys_addr_t paddr)
{
- return paddr;
+ return phys_to_machine(XPADDR(paddr)).maddr;;
}
phys_addr_t __weak xen_swiotlb_bus_to_phys(struct device *hwdev, dma_addr_t baddr)
{
- return baddr;
+ return machine_to_phys(XMADDR(baddr)).paddr;
}
static dma_addr_t xen_swiotlb_virt_to_bus(struct device *hwdev,
return !is_buffer_dma_capable(dma_get_mask(hwdev), addr, size);
}
+static int check_pages_physically_contiguous(unsigned long pfn,
+ unsigned int offset,
+ size_t length)
+{
+ unsigned long next_mfn;
+ int i;
+ int nr_pages;
+
+ next_mfn = pfn_to_mfn(pfn);
+ nr_pages = (offset + length + PAGE_SIZE-1) >> PAGE_SHIFT;
+
+ for (i = 1; i < nr_pages; i++) {
+ if (pfn_to_mfn(++pfn) != ++next_mfn)
+ return 0;
+ }
+ return 1;
+}
+
+static int range_straddles_page_boundary(phys_addr_t p, size_t size)
+{
+ unsigned long pfn = PFN_DOWN(p);
+ unsigned int offset = p & ~PAGE_MASK;
+
+ if (offset + size <= PAGE_SIZE)
+ return 0;
+ if (check_pages_physically_contiguous(pfn, offset, size))
+ return 0;
+ return 1;
+}
int __weak xen_swiotlb_arch_range_needs_mapping(phys_addr_t paddr, size_t size)
{
- return 0;
+ return range_straddles_page_boundary(paddr, size);
}
static void xen_swiotlb_print_info(unsigned long bytes)
pstart = virt_to_phys(xen_io_tlb_start);
pend = virt_to_phys(xen_io_tlb_end);
- printk(KERN_INFO "Placing %luMB software IO TLB between %p - %p\n",
+ printk(KERN_INFO "Placing %luMB Xen software IO TLB between %p - %p\n",
bytes >> 20, xen_io_tlb_start, xen_io_tlb_end);
- printk(KERN_INFO "software IO TLB at phys %#llx - %#llx\n",
+ printk(KERN_INFO "Xen software IO TLB at phys %#llx - %#llx\n",
(unsigned long long)pstart,
(unsigned long long)pend);
}
*/
xen_io_tlb_start = xen_swiotlb_alloc_boot(bytes, xen_io_tlb_nslabs);
if (!xen_io_tlb_start)
- panic("Cannot allocate SWIOTLB buffer");
+ panic("Cannot allocate Xen-SWIOTLB buffer");
xen_io_tlb_end = xen_io_tlb_start + bytes;
-
+ xen_swiotlb_fixup(xen_io_tlb_start, bytes, xen_io_tlb_nslabs);
/*
* Allocate and initialize the free list array. This array is used
* to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
xen_io_tlb_overflow_buffer = xen_swiotlb_alloc_boot(xen_io_tlb_overflow,
xen_io_tlb_overflow >> IO_TLB_SHIFT);
if (!xen_io_tlb_overflow_buffer)
- panic("Cannot allocate SWIOTLB overflow buffer!\n");
+ panic("Cannot allocate Xen-SWIOTLB overflow buffer!\n");
+
+ xen_swiotlb_fixup(xen_io_tlb_overflow_buffer, xen_io_tlb_overflow,
+ xen_io_tlb_overflow >> IO_TLB_SHIFT);
xen_swiotlb_print_info(bytes);
}
if (order != get_order(bytes)) {
printk(KERN_WARNING "Warning: only able to allocate %ld MB "
- "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
+ "for Xen software IO TLB\n", (PAGE_SIZE << order) >> 20);
xen_io_tlb_nslabs = SLABS_PER_PAGE << order;
bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT;
}
xen_io_tlb_end = xen_io_tlb_start + bytes;
+ xen_swiotlb_fixup(xen_io_tlb_start, bytes, xen_io_tlb_nslabs);
memset(xen_io_tlb_start, 0, bytes);
/*
if (!xen_io_tlb_orig_addr)
goto cleanup3;
+ xen_swiotlb_fixup(xen_io_tlb_overflow_buffer, xen_io_tlb_overflow,
+ xen_io_tlb_overflow >> IO_TLB_SHIFT);
memset(xen_io_tlb_orig_addr, 0, xen_io_tlb_nslabs * sizeof(phys_addr_t));
/*
static inline int range_needs_mapping(phys_addr_t paddr, size_t size)
{
- return xen_swiotlb_force || xen_swiotlb_arch_range_needs_mapping(paddr, size);
+ return swiotlb_force || xen_swiotlb_arch_range_needs_mapping(paddr, size);
}
static int is_xen_swiotlb_buffer(char *addr)
xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags)
{
- dma_addr_t dev_addr;
void *ret;
- int order = get_order(size);
- u64 dma_mask = DMA_BIT_MASK(32);
+ unsigned int order = get_order(size);
+ unsigned long vstart;
+ u64 mask;
- if (hwdev && hwdev->coherent_dma_mask)
- dma_mask = hwdev->coherent_dma_mask;
+ /*
+ * Ignore region specifiers - the kernel's ideas of
+ * pseudo-phys memory layout has nothing to do with the
+ * machine physical layout. We can't allocate highmem
+ * because we can't return a pointer to it.
+ */
+ flags &= ~(__GFP_DMA | __GFP_HIGHMEM);
- ret = (void *)__get_free_pages(flags, order);
- if (ret &&
- !is_buffer_dma_capable(dma_mask, xen_swiotlb_virt_to_bus(hwdev, ret),
- size)) {
- /*
- * The allocated memory isn't reachable by the device.
- */
- free_pages((unsigned long) ret, order);
- ret = NULL;
- }
- if (!ret) {
- /*
- * We are either out of memory or the device can't DMA
- * to GFP_DMA memory; fall back on map_single(), which
- * will grab memory from the lowest available address range.
- */
- ret = map_single(hwdev, 0, size, DMA_FROM_DEVICE);
- if (!ret)
- return NULL;
- }
+ if (dma_alloc_from_coherent(hwdev, size, dma_handle, &ret))
+ return ret;
- memset(ret, 0, size);
- dev_addr = xen_swiotlb_virt_to_bus(hwdev, ret);
+ vstart = __get_free_pages(flags, order);
+ ret = (void *)vstart;
- /* Confirm address can be DMA'd by device */
- if (!is_buffer_dma_capable(dma_mask, dev_addr, size)) {
- printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
- (unsigned long long)dma_mask,
- (unsigned long long)dev_addr);
+ if (hwdev != NULL && hwdev->coherent_dma_mask)
+ mask = hwdev->coherent_dma_mask;
+ else
+ mask = DMA_BIT_MASK(32);
- /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
- do_unmap_single(hwdev, ret, size, DMA_TO_DEVICE);
- return NULL;
+ if (ret != NULL) {
+ if (xen_create_contiguous_region(vstart, order,
+ fls64(mask)) != 0) {
+ free_pages(vstart, order);
+ return NULL;
+ }
+ memset(ret, 0, size);
+ *dma_handle = virt_to_machine(ret).maddr;
}
- *dma_handle = dev_addr;
return ret;
}
xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
dma_addr_t dma_handle)
{
- WARN_ON(irqs_disabled());
- if (!is_xen_swiotlb_buffer(vaddr))
- free_pages((unsigned long) vaddr, get_order(size));
- else
- /* DMA_TO_DEVICE to avoid memcpy in unmap_single */
- do_unmap_single(hwdev, vaddr, size, DMA_TO_DEVICE);
+ int order = get_order(size);
+
+ if (dma_release_from_coherent(dev, order, vaddr))
+ return;
+
+ xen_destroy_contiguous_region((unsigned long)vaddr, order);
+ free_pages((unsigned long)vaddr, order);
}
static void
* When the mapping is small enough return a static buffer to limit
* the damage, or panic when the transfer is too big.
*/
- printk(KERN_ERR "DMA: Out of SW-IOMMU space for %zu bytes at "
+ printk(KERN_ERR "Xen-DMA: Out of Xen-SWIOTLB space for %zu bytes at "
"device %s\n", size, dev ? dev_name(dev) : "?");
if (size > xen_io_tlb_overflow && do_panic) {
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
- panic("DMA: Memory would be corrupted\n");
+ panic("Xen-DMA: Memory would be corrupted\n");
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
- panic("DMA: Random memory would be DMAed\n");
+ panic("Xen-DMA: Random memory would be DMAed\n");
}
}
* Ensure that the address returned is DMA'ble
*/
if (address_needs_mapping(dev, dev_addr, size))
- panic("map_single: bounce buffer is not DMA'ble");
+ panic("xen_swiotlb_map_single: bounce buffer is not DMA'ble");
return dev_addr;
}
return xen_swiotlb_virt_to_bus(hwdev, xen_io_tlb_end - 1) <= mask;
}
+static int max_dma_bits = 32;
+
+void
+xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs)
+{
+ int i, rc;
+ int dma_bits;
+
+ printk(KERN_DEBUG "xen_swiotlb_fixup: buf=%p size=%zu\n",
+ buf, size);
+
+ dma_bits = get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT) + PAGE_SHIFT;
+
+ i = 0;
+ do {
+ int slabs = min(nslabs - i, (unsigned long)IO_TLB_SEGSIZE);
+
+ do {
+ rc = xen_create_contiguous_region(
+ (unsigned long)buf + (i << IO_TLB_SHIFT),
+ get_order(slabs << IO_TLB_SHIFT),
+ dma_bits);
+ } while (rc && dma_bits++ < max_dma_bits);
+ if (rc)
+ panic(KERN_ERR "xen_create_contiguous_region failed\n");
+
+ i += slabs;
+ } while(i < nslabs);
+}
static struct dma_map_ops xen_swiotlb_dma_ops = {
.mapping_error = xen_swiotlb_dma_mapping_error,
.alloc_coherent = xen_swiotlb_alloc_coherent,
projects / people / ssmith / netchannel2-pvops.git / commitdiff