direct-io.hg

changeset 15097:d20e4f3158c7

[IA64] Paravirtualize swiotlb

Paravirtualize our copy of swiotlb. A lot of the s/virt_to_phys/virt_to_bus
converstions have already been done in upstream, so I won't special case
them here.

Signed-off-by: Alex Williamson <alex.williamson@hp.com>
author Alex Williamson <alex.williamson@hp.com>
date Mon May 07 08:41:50 2007 -0600 (2007-05-07)
parents d4f59e652078
children a08261650b8a
files linux-2.6-xen-sparse/arch/ia64/xen/swiotlb.c
line diff
     1.1 --- a/linux-2.6-xen-sparse/arch/ia64/xen/swiotlb.c	Mon May 07 08:32:26 2007 -0600
     1.2 +++ b/linux-2.6-xen-sparse/arch/ia64/xen/swiotlb.c	Mon May 07 08:41:50 2007 -0600
     1.3 @@ -32,11 +32,22 @@
     1.4  #include <linux/init.h>
     1.5  #include <linux/bootmem.h>
     1.6  
     1.7 +#ifdef CONFIG_XEN
     1.8 +/*
     1.9 + * What DMA mask should Xen use to remap the bounce buffer pool?  Most
    1.10 + * reports seem to indicate 30 bits is sufficient, except maybe for old
    1.11 + * sound cards that we probably don't care about anyway.  If we need to,
    1.12 + * we could put in some smarts to try to lower, but hopefully it's not
    1.13 + * necessary.
    1.14 + */
    1.15 +#define DMA_BITS	(30)
    1.16 +#endif
    1.17 +
    1.18  #define OFFSET(val,align) ((unsigned long)	\
    1.19  	                   ( (val) & ( (align) - 1)))
    1.20  
    1.21  #define SG_ENT_VIRT_ADDRESS(sg)	(page_address((sg)->page) + (sg)->offset)
    1.22 -#define SG_ENT_PHYS_ADDRESS(SG)	virt_to_phys(SG_ENT_VIRT_ADDRESS(SG))
    1.23 +#define SG_ENT_PHYS_ADDRESS(SG)	virt_to_bus(SG_ENT_VIRT_ADDRESS(SG))
    1.24  
    1.25  /*
    1.26   * Maximum allowable number of contiguous slabs to map,
    1.27 @@ -139,6 +150,10 @@ swiotlb_init_with_default_size (size_t d
    1.28  		io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
    1.29  	}
    1.30  
    1.31 +#ifdef CONFIG_XEN
    1.32 +	if (is_running_on_xen())
    1.33 +		io_tlb_nslabs = roundup_pow_of_two(io_tlb_nslabs);
    1.34 +#endif
    1.35  	/*
    1.36  	 * Get IO TLB memory from the low pages
    1.37  	 */
    1.38 @@ -147,6 +162,17 @@ swiotlb_init_with_default_size (size_t d
    1.39  		panic("Cannot allocate SWIOTLB buffer");
    1.40  	io_tlb_end = io_tlb_start + io_tlb_nslabs * (1 << IO_TLB_SHIFT);
    1.41  
    1.42 +#ifdef CONFIG_XEN
    1.43 +	for (i = 0 ; i < io_tlb_nslabs ; i += IO_TLB_SEGSIZE) {
    1.44 +		if (xen_create_contiguous_region(
    1.45 +				(unsigned long)io_tlb_start +
    1.46 +				(i << IO_TLB_SHIFT),
    1.47 +				get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT),
    1.48 +				DMA_BITS))
    1.49 +			panic("Failed to setup Xen contiguous region");
    1.50 +	}
    1.51 +#endif
    1.52 +
    1.53  	/*
    1.54  	 * Allocate and initialize the free list array.  This array is used
    1.55  	 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
    1.56 @@ -162,6 +188,11 @@ swiotlb_init_with_default_size (size_t d
    1.57  	 * Get the overflow emergency buffer
    1.58  	 */
    1.59  	io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
    1.60 +#ifdef CONFIG_XEN
    1.61 +	if (xen_create_contiguous_region((unsigned long)io_tlb_overflow_buffer,
    1.62 +					 get_order(io_tlb_overflow), DMA_BITS))
    1.63 +		panic("Failed to setup Xen contiguous region for overflow");
    1.64 +#endif
    1.65  	printk(KERN_INFO "Placing software IO TLB between 0x%lx - 0x%lx\n",
    1.66  	       virt_to_phys(io_tlb_start), virt_to_phys(io_tlb_end));
    1.67  }
    1.68 @@ -188,6 +219,10 @@ swiotlb_late_init_with_default_size (siz
    1.69  		io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
    1.70  	}
    1.71  
    1.72 +#ifdef CONFIG_XEN
    1.73 +	if (is_running_on_xen())
    1.74 +		io_tlb_nslabs = roundup_pow_of_two(io_tlb_nslabs);
    1.75 +#endif
    1.76  	/*
    1.77  	 * Get IO TLB memory from the low pages
    1.78  	 */
    1.79 @@ -213,6 +248,16 @@ swiotlb_late_init_with_default_size (siz
    1.80  	io_tlb_end = io_tlb_start + io_tlb_nslabs * (1 << IO_TLB_SHIFT);
    1.81  	memset(io_tlb_start, 0, io_tlb_nslabs * (1 << IO_TLB_SHIFT));
    1.82  
    1.83 +#ifdef CONFIG_XEN
    1.84 +	for (i = 0 ; i < io_tlb_nslabs ; i += IO_TLB_SEGSIZE) {
    1.85 +		if (xen_create_contiguous_region(
    1.86 +				(unsigned long)io_tlb_start +
    1.87 +				(i << IO_TLB_SHIFT),
    1.88 +				get_order(IO_TLB_SEGSIZE << IO_TLB_SHIFT),
    1.89 +				DMA_BITS))
    1.90 +			panic("Failed to setup Xen contiguous region");
    1.91 +	}
    1.92 +#endif
    1.93  	/*
    1.94  	 * Allocate and initialize the free list array.  This array is used
    1.95  	 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
    1.96 @@ -242,6 +287,11 @@ swiotlb_late_init_with_default_size (siz
    1.97  	if (!io_tlb_overflow_buffer)
    1.98  		goto cleanup4;
    1.99  
   1.100 +#ifdef CONFIG_XEN
   1.101 +	if (xen_create_contiguous_region((unsigned long)io_tlb_overflow_buffer,
   1.102 +					 get_order(io_tlb_overflow), DMA_BITS))
   1.103 +		panic("Failed to setup Xen contiguous region for overflow");
   1.104 +#endif
   1.105  	printk(KERN_INFO "Placing %ldMB software IO TLB between 0x%lx - "
   1.106  	       "0x%lx\n", (io_tlb_nslabs * (1 << IO_TLB_SHIFT)) >> 20,
   1.107  	       virt_to_phys(io_tlb_start), virt_to_phys(io_tlb_end));
   1.108 @@ -445,7 +495,25 @@ swiotlb_alloc_coherent(struct device *hw
   1.109  	flags |= GFP_DMA;
   1.110  
   1.111  	ret = (void *)__get_free_pages(flags, order);
   1.112 -	if (ret && address_needs_mapping(hwdev, virt_to_phys(ret))) {
   1.113 +#ifdef CONFIG_XEN
   1.114 +	if (ret && is_running_on_xen()) {
   1.115 +		if (xen_create_contiguous_region((unsigned long)ret, order,
   1.116 +					fls64(hwdev->coherent_dma_mask))) {
   1.117 +			free_pages((unsigned long)ret, order);
   1.118 +			ret = NULL;
   1.119 +		} else {
   1.120 +			/*
   1.121 +			 * Short circuit the rest, xen_create_contiguous_region
   1.122 +			 * should fail if it didn't give us an address within
   1.123 +			 * the mask requested.  
   1.124 +			 */
   1.125 +			memset(ret, 0, size);
   1.126 +			*dma_handle = virt_to_bus(ret);
   1.127 +			return ret;
   1.128 +		}
   1.129 +	}
   1.130 +#endif
   1.131 +	if (ret && address_needs_mapping(hwdev, virt_to_bus(ret))) {
   1.132  		/*
   1.133  		 * The allocated memory isn't reachable by the device.
   1.134  		 * Fall back on swiotlb_map_single().
   1.135 @@ -465,11 +533,11 @@ swiotlb_alloc_coherent(struct device *hw
   1.136  		if (swiotlb_dma_mapping_error(handle))
   1.137  			return NULL;
   1.138  
   1.139 -		ret = phys_to_virt(handle);
   1.140 +		ret = bus_to_virt(handle);
   1.141  	}
   1.142  
   1.143  	memset(ret, 0, size);
   1.144 -	dev_addr = virt_to_phys(ret);
   1.145 +	dev_addr = virt_to_bus(ret);
   1.146  
   1.147  	/* Confirm address can be DMA'd by device */
   1.148  	if (address_needs_mapping(hwdev, dev_addr)) {
   1.149 @@ -487,9 +555,13 @@ swiotlb_free_coherent(struct device *hwd
   1.150  		      dma_addr_t dma_handle)
   1.151  {
   1.152  	if (!(vaddr >= (void *)io_tlb_start
   1.153 -                    && vaddr < (void *)io_tlb_end))
   1.154 +                    && vaddr < (void *)io_tlb_end)) {
   1.155 +#ifdef CONFIG_XEN
   1.156 +		xen_destroy_contiguous_region((unsigned long)vaddr,
   1.157 +					      get_order(size));
   1.158 +#endif
   1.159  		free_pages((unsigned long) vaddr, get_order(size));
   1.160 -	else
   1.161 +	} else
   1.162  		/* DMA_TO_DEVICE to avoid memcpy in unmap_single */
   1.163  		swiotlb_unmap_single (hwdev, dma_handle, size, DMA_TO_DEVICE);
   1.164  }
   1.165 @@ -525,7 +597,7 @@ swiotlb_full(struct device *dev, size_t 
   1.166  dma_addr_t
   1.167  swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
   1.168  {
   1.169 -	unsigned long dev_addr = virt_to_phys(ptr);
   1.170 +	unsigned long dev_addr = virt_to_bus(ptr);
   1.171  	void *map;
   1.172  
   1.173  	BUG_ON(dir == DMA_NONE);
   1.174 @@ -534,7 +606,8 @@ swiotlb_map_single(struct device *hwdev,
   1.175  	 * we can safely return the device addr and not worry about bounce
   1.176  	 * buffering it.
   1.177  	 */
   1.178 -	if (!address_needs_mapping(hwdev, dev_addr) && !swiotlb_force)
   1.179 +	if (!range_straddles_page_boundary(ptr, size) &&
   1.180 +	    !address_needs_mapping(hwdev, dev_addr) && !swiotlb_force)
   1.181  		return dev_addr;
   1.182  
   1.183  	/*
   1.184 @@ -546,7 +619,7 @@ swiotlb_map_single(struct device *hwdev,
   1.185  		map = io_tlb_overflow_buffer;
   1.186  	}
   1.187  
   1.188 -	dev_addr = virt_to_phys(map);
   1.189 +	dev_addr = virt_to_bus(map);
   1.190  
   1.191  	/*
   1.192  	 * Ensure that the address returned is DMA'ble
   1.193 @@ -567,6 +640,12 @@ mark_clean(void *addr, size_t size)
   1.194  {
   1.195  	unsigned long pg_addr, end;
   1.196  
   1.197 +#ifdef CONFIG_XEN
   1.198 +	/* XXX: Bad things happen when starting domUs if this is enabled. */
   1.199 +	if (is_running_on_xen())
   1.200 +		return;
   1.201 +#endif
   1.202 +
   1.203  	pg_addr = PAGE_ALIGN((unsigned long) addr);
   1.204  	end = (unsigned long) addr + size;
   1.205  	while (pg_addr + PAGE_SIZE <= end) {
   1.206 @@ -588,7 +667,7 @@ void
   1.207  swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
   1.208  		     int dir)
   1.209  {
   1.210 -	char *dma_addr = phys_to_virt(dev_addr);
   1.211 +	char *dma_addr = bus_to_virt(dev_addr);
   1.212  
   1.213  	BUG_ON(dir == DMA_NONE);
   1.214  	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
   1.215 @@ -611,7 +690,7 @@ static inline void
   1.216  swiotlb_sync_single(struct device *hwdev, dma_addr_t dev_addr,
   1.217  		    size_t size, int dir, int target)
   1.218  {
   1.219 -	char *dma_addr = phys_to_virt(dev_addr);
   1.220 +	char *dma_addr = bus_to_virt(dev_addr);
   1.221  
   1.222  	BUG_ON(dir == DMA_NONE);
   1.223  	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
   1.224 @@ -642,7 +721,7 @@ swiotlb_sync_single_range(struct device 
   1.225  			  unsigned long offset, size_t size,
   1.226  			  int dir, int target)
   1.227  {
   1.228 -	char *dma_addr = phys_to_virt(dev_addr) + offset;
   1.229 +	char *dma_addr = bus_to_virt(dev_addr) + offset;
   1.230  
   1.231  	BUG_ON(dir == DMA_NONE);
   1.232  	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
   1.233 @@ -695,7 +774,7 @@ swiotlb_map_sg(struct device *hwdev, str
   1.234  
   1.235  	for (i = 0; i < nelems; i++, sg++) {
   1.236  		addr = SG_ENT_VIRT_ADDRESS(sg);
   1.237 -		dev_addr = virt_to_phys(addr);
   1.238 +		dev_addr = virt_to_bus(addr);
   1.239  		if (swiotlb_force || address_needs_mapping(hwdev, dev_addr)) {
   1.240  			void *map = map_single(hwdev, addr, sg->length, dir);
   1.241  			sg->dma_address = virt_to_bus(map);
   1.242 @@ -728,7 +807,7 @@ swiotlb_unmap_sg(struct device *hwdev, s
   1.243  
   1.244  	for (i = 0; i < nelems; i++, sg++)
   1.245  		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
   1.246 -			unmap_single(hwdev, (void *) phys_to_virt(sg->dma_address), sg->dma_length, dir);
   1.247 +			unmap_single(hwdev, (void *) bus_to_virt(sg->dma_address), sg->dma_length, dir);
   1.248  		else if (dir == DMA_FROM_DEVICE)
   1.249  			mark_clean(SG_ENT_VIRT_ADDRESS(sg), sg->dma_length);
   1.250  }
   1.251 @@ -771,7 +850,7 @@ swiotlb_sync_sg_for_device(struct device
   1.252  int
   1.253  swiotlb_dma_mapping_error(dma_addr_t dma_addr)
   1.254  {
   1.255 -	return (dma_addr == virt_to_phys(io_tlb_overflow_buffer));
   1.256 +	return (dma_addr == virt_to_bus(io_tlb_overflow_buffer));
   1.257  }
   1.258  
   1.259  /*
   1.260 @@ -783,7 +862,7 @@ swiotlb_dma_mapping_error(dma_addr_t dma
   1.261  int
   1.262  swiotlb_dma_supported (struct device *hwdev, u64 mask)
   1.263  {
   1.264 -	return (virt_to_phys (io_tlb_end) - 1) <= mask;
   1.265 +	return (virt_to_bus(io_tlb_end) - 1) <= mask;
   1.266  }
   1.267  
   1.268  EXPORT_SYMBOL(swiotlb_init);