ia64/xen-unstable

changeset 6207:3d187585c141

Use a SWIOTLB to use pre-reserved bounce buffers for high memory
and multi-page DMA accesses. This is based on a preliminary patch
from Suresh Siddha at Intel.

Signed-off-by: Keir Fraser <keir@xensource.com>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
author kaf24@firebug.cl.cam.ac.uk
date Tue Aug 16 11:20:47 2005 +0000 (2005-08-16)
parents eaf498f1ffde
children d80dd1169acc 7d8ed9e538ff
files linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32 linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64 linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32 linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64 linux-2.6-xen-sparse/arch/xen/i386/Kconfig linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c linux-2.6-xen-sparse/arch/xen/i386/kernel/swiotlb.c linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c linux-2.6-xen-sparse/arch/xen/i386/mm/init.c linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h linux-2.6-xen-sparse/include/asm-xen/asm-i386/kmap_types.h linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h linux-2.6-xen-sparse/include/asm-xen/asm-i386/scatterlist.h linux-2.6-xen-sparse/include/asm-xen/asm-i386/swiotlb.h linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h linux-2.6-xen-sparse/include/asm-xen/hypervisor.h
line diff
     1.1 --- a/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32	Tue Aug 16 10:12:18 2005 +0000
     1.2 +++ b/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_32	Tue Aug 16 11:20:47 2005 +0000
     1.3 @@ -130,6 +130,7 @@ CONFIG_PREEMPT_BKL=y
     1.4  # CONFIG_X86_REBOOTFIXUPS is not set
     1.5  CONFIG_MICROCODE=y
     1.6  CONFIG_X86_CPUID=y
     1.7 +CONFIG_SWIOTLB=y
     1.8  
     1.9  #
    1.10  # Firmware Drivers
     2.1 --- a/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64	Tue Aug 16 10:12:18 2005 +0000
     2.2 +++ b/linux-2.6-xen-sparse/arch/xen/configs/xen0_defconfig_x86_64	Tue Aug 16 11:20:47 2005 +0000
     2.3 @@ -123,6 +123,7 @@ CONFIG_X86_XEN_GENAPIC=y
     2.4  # CONFIG_X86_MSR is not set
     2.5  # CONFIG_GART_IOMMU is not set
     2.6  CONFIG_DUMMY_IOMMU=y
     2.7 +CONFIG_SWIOTLB=y
     2.8  # CONFIG_X86_MCE is not set
     2.9  
    2.10  #
     3.1 --- a/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32	Tue Aug 16 10:12:18 2005 +0000
     3.2 +++ b/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_32	Tue Aug 16 11:20:47 2005 +0000
     3.3 @@ -137,6 +137,7 @@ CONFIG_PREEMPT_BKL=y
     3.4  # CONFIG_X86_REBOOTFIXUPS is not set
     3.5  CONFIG_MICROCODE=m
     3.6  CONFIG_X86_CPUID=m
     3.7 +CONFIG_SWIOTLB=y
     3.8  
     3.9  #
    3.10  # Firmware Drivers
     4.1 --- a/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64	Tue Aug 16 10:12:18 2005 +0000
     4.2 +++ b/linux-2.6-xen-sparse/arch/xen/configs/xen_defconfig_x86_64	Tue Aug 16 11:20:47 2005 +0000
     4.3 @@ -132,6 +132,7 @@ CONFIG_X86_HT=y
     4.4  # CONFIG_NUMA_EMU is not set
     4.5  # CONFIG_GART_IOMMU is not set
     4.6  CONFIG_DUMMY_IOMMU=y
     4.7 +CONFIG_SWIOTLB=y
     4.8  # CONFIG_X86_MCE is not set
     4.9  
    4.10  #
     5.1 --- a/linux-2.6-xen-sparse/arch/xen/i386/Kconfig	Tue Aug 16 10:12:18 2005 +0000
     5.2 +++ b/linux-2.6-xen-sparse/arch/xen/i386/Kconfig	Tue Aug 16 11:20:47 2005 +0000
     5.3 @@ -533,6 +533,11 @@ config X86_CPUID
     5.4  	  with major 203 and minors 0 to 31 for /dev/cpu/0/cpuid to
     5.5  	  /dev/cpu/31/cpuid.
     5.6  
     5.7 +config SWIOTLB
     5.8 +       bool
     5.9 +       depends on PCI
    5.10 +       default y
    5.11 +
    5.12  source "drivers/firmware/Kconfig"
    5.13  
    5.14  choice
     6.1 --- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile	Tue Aug 16 10:12:18 2005 +0000
     6.2 +++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/Makefile	Tue Aug 16 11:20:47 2005 +0000
     6.3 @@ -44,6 +44,7 @@ c-obj-$(CONFIG_HPET_TIMER) 	+= time_hpet
     6.4  c-obj-$(CONFIG_EFI) 		+= efi.o efi_stub.o
     6.5  c-obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
     6.6  c-obj-$(CONFIG_SMP_ALTERNATIVES)+= smpalts.o
     6.7 +c-obj-$(CONFIG_SWIOTLB)		+= swiotlb.o
     6.8  
     6.9  EXTRA_AFLAGS   := -traditional
    6.10  
     7.1 --- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c	Tue Aug 16 10:12:18 2005 +0000
     7.2 +++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c	Tue Aug 16 11:20:47 2005 +0000
     7.3 @@ -24,6 +24,103 @@ struct dma_coherent_mem {
     7.4  	unsigned long	*bitmap;
     7.5  };
     7.6  
     7.7 +static void iommu_bug(void)
     7.8 +{
     7.9 +	printk(KERN_ALERT "Fatal DMA error! Please use 'swiotlb=force'\n");
    7.10 +	BUG();
    7.11 +}
    7.12 +
    7.13 +#define IOMMU_BUG_ON(test) do { if (unlikely(test)) iommu_bug(); } while(0)
    7.14 +
    7.15 +int
    7.16 +dma_map_sg(struct device *hwdev, struct scatterlist *sg, int nents,
    7.17 +	   enum dma_data_direction direction)
    7.18 +{
    7.19 +	int i, rc;
    7.20 +
    7.21 +	BUG_ON(direction == DMA_NONE);
    7.22 +
    7.23 +	if (swiotlb) {
    7.24 +		rc = swiotlb_map_sg(hwdev, sg, nents, direction);
    7.25 +	} else {
    7.26 +		for (i = 0; i < nents; i++ ) {
    7.27 +			sg[i].dma_address =
    7.28 +				page_to_phys(sg[i].page) + sg[i].offset;
    7.29 +			sg[i].dma_length  = sg[i].length;
    7.30 +			BUG_ON(!sg[i].page);
    7.31 +			IOMMU_BUG_ON(address_needs_mapping(
    7.32 +				hwdev, sg[i].dma_address));
    7.33 +		}
    7.34 +		rc = nents;
    7.35 +	}
    7.36 +
    7.37 +	flush_write_buffers();
    7.38 +	return rc;
    7.39 +}
    7.40 +EXPORT_SYMBOL(dma_map_sg);
    7.41 +
    7.42 +void
    7.43 +dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
    7.44 +	     enum dma_data_direction direction)
    7.45 +{
    7.46 +	BUG_ON(direction == DMA_NONE);
    7.47 +	if (swiotlb)
    7.48 +		swiotlb_unmap_sg(hwdev, sg, nents, direction);
    7.49 +}
    7.50 +EXPORT_SYMBOL(dma_unmap_sg);
    7.51 +
    7.52 +dma_addr_t
    7.53 +dma_map_page(struct device *dev, struct page *page, unsigned long offset,
    7.54 +	     size_t size, enum dma_data_direction direction)
    7.55 +{
    7.56 +	dma_addr_t dma_addr;
    7.57 +
    7.58 +	BUG_ON(direction == DMA_NONE);
    7.59 +
    7.60 +	if (swiotlb) {
    7.61 +		dma_addr = swiotlb_map_page(
    7.62 +			dev, page, offset, size, direction);
    7.63 +	} else {
    7.64 +		dma_addr = page_to_phys(page) + offset;
    7.65 +		IOMMU_BUG_ON(address_needs_mapping(dev, dma_addr));
    7.66 +	}
    7.67 +
    7.68 +	return dma_addr;
    7.69 +}
    7.70 +EXPORT_SYMBOL(dma_map_page);
    7.71 +
    7.72 +void
    7.73 +dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
    7.74 +	       enum dma_data_direction direction)
    7.75 +{
    7.76 +	BUG_ON(direction == DMA_NONE);
    7.77 +	if (swiotlb)
    7.78 +		swiotlb_unmap_page(dev, dma_address, size, direction);
    7.79 +}
    7.80 +EXPORT_SYMBOL(dma_unmap_page);
    7.81 +
    7.82 +int
    7.83 +dma_mapping_error(dma_addr_t dma_addr)
    7.84 +{
    7.85 +	if (swiotlb)
    7.86 +		return swiotlb_dma_mapping_error(dma_addr);
    7.87 +	return 0;
    7.88 +}
    7.89 +EXPORT_SYMBOL(dma_mapping_error);
    7.90 +
    7.91 +int
    7.92 +dma_supported(struct device *dev, u64 mask)
    7.93 +{
    7.94 +	if (swiotlb)
    7.95 +		return swiotlb_dma_supported(dev, mask);
    7.96 +	/*
    7.97 +         * By default we'll BUG when an infeasible DMA is requested, and
    7.98 +         * request swiotlb=force (see IOMMU_BUG_ON).
    7.99 +         */
   7.100 +	return 1;
   7.101 +}
   7.102 +EXPORT_SYMBOL(dma_supported);
   7.103 +
   7.104  void *dma_alloc_coherent(struct device *dev, size_t size,
   7.105  			   dma_addr_t *dma_handle, unsigned int __nocast gfp)
   7.106  {
   7.107 @@ -54,13 +151,14 @@ void *dma_alloc_coherent(struct device *
   7.108  	ret = (void *)vstart;
   7.109  
   7.110  	if (ret != NULL) {
   7.111 -		xen_contig_memory(vstart, order);
   7.112 +		xen_create_contiguous_region(vstart, order);
   7.113  
   7.114  		memset(ret, 0, size);
   7.115  		*dma_handle = virt_to_bus(ret);
   7.116  	}
   7.117  	return ret;
   7.118  }
   7.119 +EXPORT_SYMBOL(dma_alloc_coherent);
   7.120  
   7.121  void dma_free_coherent(struct device *dev, size_t size,
   7.122  			 void *vaddr, dma_addr_t dma_handle)
   7.123 @@ -72,9 +170,12 @@ void dma_free_coherent(struct device *de
   7.124  		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
   7.125  
   7.126  		bitmap_release_region(mem->bitmap, page, order);
   7.127 -	} else
   7.128 +	} else {
   7.129 +		xen_destroy_contiguous_region((unsigned long)vaddr, order);
   7.130  		free_pages((unsigned long)vaddr, order);
   7.131 +	}
   7.132  }
   7.133 +EXPORT_SYMBOL(dma_free_coherent);
   7.134  
   7.135  int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
   7.136  				dma_addr_t device_addr, size_t size, int flags)
   7.137 @@ -153,46 +254,20 @@ void *dma_mark_declared_memory_occupied(
   7.138  }
   7.139  EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
   7.140  
   7.141 -static LIST_HEAD(dma_map_head);
   7.142 -static DEFINE_SPINLOCK(dma_map_lock);
   7.143 -struct dma_map_entry {
   7.144 -	struct list_head list;
   7.145 -	dma_addr_t dma;
   7.146 -	char *bounce, *host;
   7.147 -	size_t size;
   7.148 -};
   7.149 -#define DMA_MAP_MATCHES(e,d) (((e)->dma<=(d)) && (((e)->dma+(e)->size)>(d)))
   7.150 -
   7.151  dma_addr_t
   7.152  dma_map_single(struct device *dev, void *ptr, size_t size,
   7.153  	       enum dma_data_direction direction)
   7.154  {
   7.155 -	struct dma_map_entry *ent;
   7.156 -	void *bnc;
   7.157  	dma_addr_t dma;
   7.158 -	unsigned long flags;
   7.159  
   7.160  	BUG_ON(direction == DMA_NONE);
   7.161  
   7.162 -	/*
   7.163 -	 * Even if size is sub-page, the buffer may still straddle a page
   7.164 -	 * boundary. Take into account buffer start offset. All other calls are
   7.165 -	 * conservative and always search the dma_map list if it's non-empty.
   7.166 -	 */
   7.167 -	if ((((unsigned int)ptr & ~PAGE_MASK) + size) <= PAGE_SIZE) {
   7.168 -		dma = virt_to_bus(ptr);
   7.169 +	if (swiotlb) {
   7.170 +		dma = swiotlb_map_single(dev, ptr, size, direction);
   7.171  	} else {
   7.172 -		BUG_ON((bnc = dma_alloc_coherent(dev, size, &dma, GFP_ATOMIC)) == NULL);
   7.173 -		BUG_ON((ent = kmalloc(sizeof(*ent), GFP_ATOMIC)) == NULL);
   7.174 -		if (direction != DMA_FROM_DEVICE)
   7.175 -			memcpy(bnc, ptr, size);
   7.176 -		ent->dma    = dma;
   7.177 -		ent->bounce = bnc;
   7.178 -		ent->host   = ptr;
   7.179 -		ent->size   = size;
   7.180 -		spin_lock_irqsave(&dma_map_lock, flags);
   7.181 -		list_add(&ent->list, &dma_map_head);
   7.182 -		spin_unlock_irqrestore(&dma_map_lock, flags);
   7.183 +		dma = virt_to_bus(ptr);
   7.184 +		IOMMU_BUG_ON(range_straddles_page_boundary(ptr, size));
   7.185 +		IOMMU_BUG_ON(address_needs_mapping(dev, dma));
   7.186  	}
   7.187  
   7.188  	flush_write_buffers();
   7.189 @@ -204,30 +279,9 @@ void
   7.190  dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
   7.191  		 enum dma_data_direction direction)
   7.192  {
   7.193 -	struct dma_map_entry *ent;
   7.194 -	unsigned long flags;
   7.195 -
   7.196  	BUG_ON(direction == DMA_NONE);
   7.197 -
   7.198 -	/* Fast-path check: are there any multi-page DMA mappings? */
   7.199 -	if (!list_empty(&dma_map_head)) {
   7.200 -		spin_lock_irqsave(&dma_map_lock, flags);
   7.201 -		list_for_each_entry ( ent, &dma_map_head, list ) {
   7.202 -			if (DMA_MAP_MATCHES(ent, dma_addr)) {
   7.203 -				list_del(&ent->list);
   7.204 -				break;
   7.205 -			}
   7.206 -		}
   7.207 -		spin_unlock_irqrestore(&dma_map_lock, flags);
   7.208 -		if (&ent->list != &dma_map_head) {
   7.209 -			BUG_ON(dma_addr != ent->dma);
   7.210 -			BUG_ON(size != ent->size);
   7.211 -			if (direction != DMA_TO_DEVICE)
   7.212 -				memcpy(ent->host, ent->bounce, size);
   7.213 -			dma_free_coherent(dev, size, ent->bounce, ent->dma);
   7.214 -			kfree(ent);
   7.215 -		}
   7.216 -	}
   7.217 +	if (swiotlb)
   7.218 +		swiotlb_unmap_single(dev, dma_addr, size, direction);
   7.219  }
   7.220  EXPORT_SYMBOL(dma_unmap_single);
   7.221  
   7.222 @@ -235,23 +289,8 @@ void
   7.223  dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
   7.224  			enum dma_data_direction direction)
   7.225  {
   7.226 -	struct dma_map_entry *ent;
   7.227 -	unsigned long flags, off;
   7.228 -
   7.229 -	/* Fast-path check: are there any multi-page DMA mappings? */
   7.230 -	if (!list_empty(&dma_map_head)) {
   7.231 -		spin_lock_irqsave(&dma_map_lock, flags);
   7.232 -		list_for_each_entry ( ent, &dma_map_head, list )
   7.233 -			if (DMA_MAP_MATCHES(ent, dma_handle))
   7.234 -				break;
   7.235 -		spin_unlock_irqrestore(&dma_map_lock, flags);
   7.236 -		if (&ent->list != &dma_map_head) {
   7.237 -			off = dma_handle - ent->dma;
   7.238 -			BUG_ON((off + size) > ent->size);
   7.239 -			/*if (direction != DMA_TO_DEVICE)*/
   7.240 -				memcpy(ent->host+off, ent->bounce+off, size);
   7.241 -		}
   7.242 -	}
   7.243 +	if (swiotlb)
   7.244 +		swiotlb_sync_single_for_cpu(dev, dma_handle, size, direction);
   7.245  }
   7.246  EXPORT_SYMBOL(dma_sync_single_for_cpu);
   7.247  
   7.248 @@ -259,24 +298,17 @@ void
   7.249  dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size,
   7.250                             enum dma_data_direction direction)
   7.251  {
   7.252 -	struct dma_map_entry *ent;
   7.253 -	unsigned long flags, off;
   7.254 -
   7.255 -	/* Fast-path check: are there any multi-page DMA mappings? */
   7.256 -	if (!list_empty(&dma_map_head)) {
   7.257 -		spin_lock_irqsave(&dma_map_lock, flags);
   7.258 -		list_for_each_entry ( ent, &dma_map_head, list )
   7.259 -			if (DMA_MAP_MATCHES(ent, dma_handle))
   7.260 -				break;
   7.261 -		spin_unlock_irqrestore(&dma_map_lock, flags);
   7.262 -		if (&ent->list != &dma_map_head) {
   7.263 -			off = dma_handle - ent->dma;
   7.264 -			BUG_ON((off + size) > ent->size);
   7.265 -			/*if (direction != DMA_FROM_DEVICE)*/
   7.266 -				memcpy(ent->bounce+off, ent->host+off, size);
   7.267 -		}
   7.268 -	}
   7.269 -
   7.270 -	flush_write_buffers();
   7.271 +	if (swiotlb)
   7.272 +		swiotlb_sync_single_for_device(dev, dma_handle, size, direction);
   7.273  }
   7.274  EXPORT_SYMBOL(dma_sync_single_for_device);
   7.275 +
   7.276 +/*
   7.277 + * Local variables:
   7.278 + *  c-file-style: "linux"
   7.279 + *  indent-tabs-mode: t
   7.280 + *  c-indent-level: 8
   7.281 + *  c-basic-offset: 8
   7.282 + *  tab-width: 8
   7.283 + * End:
   7.284 + */
     8.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     8.2 +++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/swiotlb.c	Tue Aug 16 11:20:47 2005 +0000
     8.3 @@ -0,0 +1,653 @@
     8.4 +/*
     8.5 + * Dynamic DMA mapping support.
     8.6 + *
     8.7 + * This implementation is a fallback for platforms that do not support
     8.8 + * I/O TLBs (aka DMA address translation hardware).
     8.9 + * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
    8.10 + * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
    8.11 + * Copyright (C) 2000, 2003 Hewlett-Packard Co
    8.12 + *	David Mosberger-Tang <davidm@hpl.hp.com>
    8.13 + * Copyright (C) 2005 Keir Fraser <keir@xensource.com>
    8.14 + */
    8.15 +
    8.16 +#include <linux/cache.h>
    8.17 +#include <linux/mm.h>
    8.18 +#include <linux/module.h>
    8.19 +#include <linux/pci.h>
    8.20 +#include <linux/spinlock.h>
    8.21 +#include <linux/string.h>
    8.22 +#include <linux/types.h>
    8.23 +#include <linux/ctype.h>
    8.24 +#include <linux/init.h>
    8.25 +#include <linux/bootmem.h>
    8.26 +#include <linux/highmem.h>
    8.27 +#include <asm/io.h>
    8.28 +#include <asm/pci.h>
    8.29 +#include <asm/dma.h>
    8.30 +
    8.31 +#define OFFSET(val,align) ((unsigned long)((val) & ( (align) - 1)))
    8.32 +
    8.33 +#define SG_ENT_PHYS_ADDRESS(sg)	(page_to_phys((sg)->page) + (sg)->offset)
    8.34 +
    8.35 +/*
    8.36 + * Maximum allowable number of contiguous slabs to map,
    8.37 + * must be a power of 2.  What is the appropriate value ?
    8.38 + * The complexity of {map,unmap}_single is linearly dependent on this value.
    8.39 + */
    8.40 +#define IO_TLB_SEGSIZE	128
    8.41 +
    8.42 +/*
    8.43 + * log of the size of each IO TLB slab.  The number of slabs is command line
    8.44 + * controllable.
    8.45 + */
    8.46 +#define IO_TLB_SHIFT 11
    8.47 +
    8.48 +int swiotlb_force;
    8.49 +
    8.50 +/*
    8.51 + * Used to do a quick range check in swiotlb_unmap_single and
    8.52 + * swiotlb_sync_single_*, to see if the memory was in fact allocated by this
    8.53 + * API.
    8.54 + */
    8.55 +static char *io_tlb_start, *io_tlb_end;
    8.56 +
    8.57 +/*
    8.58 + * The number of IO TLB blocks (in groups of 64) betweeen io_tlb_start and
    8.59 + * io_tlb_end.  This is command line adjustable via setup_io_tlb_npages.
    8.60 + */
    8.61 +static unsigned long io_tlb_nslabs;
    8.62 +
    8.63 +/*
    8.64 + * When the IOMMU overflows we return a fallback buffer. This sets the size.
    8.65 + */
    8.66 +static unsigned long io_tlb_overflow = 32*1024;
    8.67 +
    8.68 +void *io_tlb_overflow_buffer;
    8.69 +
    8.70 +/*
    8.71 + * This is a free list describing the number of free entries available from
    8.72 + * each index
    8.73 + */
    8.74 +static unsigned int *io_tlb_list;
    8.75 +static unsigned int io_tlb_index;
    8.76 +
    8.77 +/*
    8.78 + * We need to save away the original address corresponding to a mapped entry
    8.79 + * for the sync operations.
    8.80 + */
    8.81 +static struct phys_addr {
    8.82 +	struct page *page;
    8.83 +	unsigned int offset;
    8.84 +} *io_tlb_orig_addr;
    8.85 +
    8.86 +/*
    8.87 + * Protect the above data structures in the map and unmap calls
    8.88 + */
    8.89 +static DEFINE_SPINLOCK(io_tlb_lock);
    8.90 +
    8.91 +static int __init
    8.92 +setup_io_tlb_npages(char *str)
    8.93 +{
    8.94 +	if (isdigit(*str)) {
    8.95 +		io_tlb_nslabs = simple_strtoul(str, &str, 0) <<
    8.96 +			(PAGE_SHIFT - IO_TLB_SHIFT);
    8.97 +		/* avoid tail segment of size < IO_TLB_SEGSIZE */
    8.98 +		io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
    8.99 +	}
   8.100 +	if (*str == ',')
   8.101 +		++str;
   8.102 +	/*
   8.103 +         * NB. 'force' enables the swiotlb, but doesn't force its use for
   8.104 +         * every DMA like it does on native Linux.
   8.105 +         */
   8.106 +	if (!strcmp(str, "force"))
   8.107 +		swiotlb_force = 1;
   8.108 +	return 1;
   8.109 +}
   8.110 +__setup("swiotlb=", setup_io_tlb_npages);
   8.111 +/* make io_tlb_overflow tunable too? */
   8.112 +
   8.113 +/*
   8.114 + * Statically reserve bounce buffer space and initialize bounce buffer data
   8.115 + * structures for the software IO TLB used to implement the PCI DMA API.
   8.116 + */
   8.117 +void
   8.118 +swiotlb_init_with_default_size (size_t default_size)
   8.119 +{
   8.120 +	unsigned long i;
   8.121 +
   8.122 +	if (!io_tlb_nslabs) {
   8.123 +		io_tlb_nslabs = (default_size >> PAGE_SHIFT);
   8.124 +		io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
   8.125 +	}
   8.126 +
   8.127 +	/*
   8.128 +	 * Get IO TLB memory from the low pages
   8.129 +	 */
   8.130 +	io_tlb_start = alloc_bootmem_low_pages(io_tlb_nslabs *
   8.131 +					       (1 << IO_TLB_SHIFT));
   8.132 +	if (!io_tlb_start)
   8.133 +		panic("Cannot allocate SWIOTLB buffer");
   8.134 +
   8.135 +	xen_create_contiguous_region(
   8.136 +		(unsigned long)io_tlb_start, 
   8.137 +		get_order(io_tlb_nslabs * (1 << IO_TLB_SHIFT)));
   8.138 +
   8.139 +	io_tlb_end = io_tlb_start + io_tlb_nslabs * (1 << IO_TLB_SHIFT);
   8.140 +
   8.141 +	/*
   8.142 +	 * Allocate and initialize the free list array.  This array is used
   8.143 +	 * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
   8.144 +	 * between io_tlb_start and io_tlb_end.
   8.145 +	 */
   8.146 +	io_tlb_list = alloc_bootmem(io_tlb_nslabs * sizeof(int));
   8.147 +	for (i = 0; i < io_tlb_nslabs; i++)
   8.148 + 		io_tlb_list[i] = IO_TLB_SEGSIZE - OFFSET(i, IO_TLB_SEGSIZE);
   8.149 +	io_tlb_index = 0;
   8.150 +	io_tlb_orig_addr = alloc_bootmem(
   8.151 +		io_tlb_nslabs * sizeof(*io_tlb_orig_addr));
   8.152 +
   8.153 +	/*
   8.154 +	 * Get the overflow emergency buffer
   8.155 +	 */
   8.156 +	io_tlb_overflow_buffer = alloc_bootmem_low(io_tlb_overflow);
   8.157 +	printk(KERN_INFO "Placing software IO TLB between 0x%lx - 0x%lx\n",
   8.158 +	       virt_to_bus(io_tlb_start), virt_to_bus(io_tlb_end-1));
   8.159 +}
   8.160 +
   8.161 +void
   8.162 +swiotlb_init(void)
   8.163 +{
   8.164 +	/* The user can forcibly enable swiotlb. */
   8.165 +	if (swiotlb_force)
   8.166 +		swiotlb = 1;
   8.167 +
   8.168 +	/*
   8.169 +         * Otherwise, enable for domain 0 if the machine has 'lots of memory',
   8.170 +         * which we take to mean more than 2GB.
   8.171 +         */
   8.172 +	if (xen_start_info.flags & SIF_INITDOMAIN) {
   8.173 +		dom0_op_t op;
   8.174 +		op.cmd = DOM0_PHYSINFO;
   8.175 +		if ((HYPERVISOR_dom0_op(&op) == 0) &&
   8.176 +		    (op.u.physinfo.total_pages > 0x7ffff))
   8.177 +			swiotlb = 1;
   8.178 +	}
   8.179 +
   8.180 +	if (swiotlb)
   8.181 +		swiotlb_init_with_default_size(64 * (1<<20));
   8.182 +}
   8.183 +
   8.184 +static void
   8.185 +__sync_single(struct phys_addr buffer, char *dma_addr, size_t size, int dir)
   8.186 +{
   8.187 +	if (PageHighMem(buffer.page)) {
   8.188 +		size_t len, bytes;
   8.189 +		char *dev, *host, *kmp;
   8.190 +		len = size;
   8.191 +		while (len != 0) {
   8.192 +			if (((bytes = len) + buffer.offset) > PAGE_SIZE)
   8.193 +				bytes = PAGE_SIZE - buffer.offset;
   8.194 +			kmp  = kmap_atomic(buffer.page, KM_SWIOTLB);
   8.195 +			dev  = dma_addr + size - len;
   8.196 +			host = kmp + buffer.offset;
   8.197 +			memcpy((dir == DMA_FROM_DEVICE) ? host : dev,
   8.198 +			       (dir == DMA_FROM_DEVICE) ? dev : host,
   8.199 +			       bytes);
   8.200 +			kunmap_atomic(kmp, KM_SWIOTLB);
   8.201 +			len -= bytes;
   8.202 +			buffer.page++;
   8.203 +			buffer.offset = 0;
   8.204 +		}
   8.205 +	} else {
   8.206 +		char *host = (char *)phys_to_virt(
   8.207 +			page_to_pseudophys(buffer.page)) + buffer.offset;
   8.208 +		if (dir == DMA_FROM_DEVICE)
   8.209 +			memcpy(host, dma_addr, size);
   8.210 +		else if (dir == DMA_TO_DEVICE)
   8.211 +			memcpy(dma_addr, host, size);
   8.212 +	}
   8.213 +}
   8.214 +
   8.215 +/*
   8.216 + * Allocates bounce buffer and returns its kernel virtual address.
   8.217 + */
   8.218 +static void *
   8.219 +map_single(struct device *hwdev, struct phys_addr buffer, size_t size, int dir)
   8.220 +{
   8.221 +	unsigned long flags;
   8.222 +	char *dma_addr;
   8.223 +	unsigned int nslots, stride, index, wrap;
   8.224 +	int i;
   8.225 +
   8.226 +	/*
   8.227 +	 * For mappings greater than a page, we limit the stride (and
   8.228 +	 * hence alignment) to a page size.
   8.229 +	 */
   8.230 +	nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
   8.231 +	if (size > PAGE_SIZE)
   8.232 +		stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
   8.233 +	else
   8.234 +		stride = 1;
   8.235 +
   8.236 +	BUG_ON(!nslots);
   8.237 +
   8.238 +	/*
   8.239 +	 * Find suitable number of IO TLB entries size that will fit this
   8.240 +	 * request and allocate a buffer from that IO TLB pool.
   8.241 +	 */
   8.242 +	spin_lock_irqsave(&io_tlb_lock, flags);
   8.243 +	{
   8.244 +		wrap = index = ALIGN(io_tlb_index, stride);
   8.245 +
   8.246 +		if (index >= io_tlb_nslabs)
   8.247 +			wrap = index = 0;
   8.248 +
   8.249 +		do {
   8.250 +			/*
   8.251 +			 * If we find a slot that indicates we have 'nslots'
   8.252 +			 * number of contiguous buffers, we allocate the
   8.253 +			 * buffers from that slot and mark the entries as '0'
   8.254 +			 * indicating unavailable.
   8.255 +			 */
   8.256 +			if (io_tlb_list[index] >= nslots) {
   8.257 +				int count = 0;
   8.258 +
   8.259 +				for (i = index; i < (int)(index + nslots); i++)
   8.260 +					io_tlb_list[i] = 0;
   8.261 +				for (i = index - 1;
   8.262 +				     (OFFSET(i, IO_TLB_SEGSIZE) !=
   8.263 +				      IO_TLB_SEGSIZE -1) && io_tlb_list[i];
   8.264 +				     i--)
   8.265 +					io_tlb_list[i] = ++count;
   8.266 +				dma_addr = io_tlb_start +
   8.267 +					(index << IO_TLB_SHIFT);
   8.268 +
   8.269 +				/*
   8.270 +				 * Update the indices to avoid searching in
   8.271 +				 * the next round.
   8.272 +				 */
   8.273 +				io_tlb_index = 
   8.274 +					((index + nslots) < io_tlb_nslabs
   8.275 +					 ? (index + nslots) : 0);
   8.276 +
   8.277 +				goto found;
   8.278 +			}
   8.279 +			index += stride;
   8.280 +			if (index >= io_tlb_nslabs)
   8.281 +				index = 0;
   8.282 +		} while (index != wrap);
   8.283 +
   8.284 +		spin_unlock_irqrestore(&io_tlb_lock, flags);
   8.285 +		return NULL;
   8.286 +	}
   8.287 +  found:
   8.288 +	spin_unlock_irqrestore(&io_tlb_lock, flags);
   8.289 +
   8.290 +	/*
   8.291 +	 * Save away the mapping from the original address to the DMA address.
   8.292 +	 * This is needed when we sync the memory.  Then we sync the buffer if
   8.293 +	 * needed.
   8.294 +	 */
   8.295 +	io_tlb_orig_addr[index] = buffer;
   8.296 +	if ((dir == DMA_TO_DEVICE) || (dir == DMA_BIDIRECTIONAL))
   8.297 +		__sync_single(buffer, dma_addr, size, DMA_TO_DEVICE);
   8.298 +
   8.299 +	return dma_addr;
   8.300 +}
   8.301 +
   8.302 +/*
   8.303 + * dma_addr is the kernel virtual address of the bounce buffer to unmap.
   8.304 + */
   8.305 +static void
   8.306 +unmap_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
   8.307 +{
   8.308 +	unsigned long flags;
   8.309 +	int i, count, nslots = ALIGN(size, 1 << IO_TLB_SHIFT) >> IO_TLB_SHIFT;
   8.310 +	int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
   8.311 +	struct phys_addr buffer = io_tlb_orig_addr[index];
   8.312 +
   8.313 +	/*
   8.314 +	 * First, sync the memory before unmapping the entry
   8.315 +	 */
   8.316 +	if ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL))
   8.317 +		__sync_single(buffer, dma_addr, size, DMA_FROM_DEVICE);
   8.318 +
   8.319 +	/*
   8.320 +	 * Return the buffer to the free list by setting the corresponding
   8.321 +	 * entries to indicate the number of contigous entries available.
   8.322 +	 * While returning the entries to the free list, we merge the entries
   8.323 +	 * with slots below and above the pool being returned.
   8.324 +	 */
   8.325 +	spin_lock_irqsave(&io_tlb_lock, flags);
   8.326 +	{
   8.327 +		count = ((index + nslots) < ALIGN(index + 1, IO_TLB_SEGSIZE) ?
   8.328 +			 io_tlb_list[index + nslots] : 0);
   8.329 +		/*
   8.330 +		 * Step 1: return the slots to the free list, merging the
   8.331 +		 * slots with superceeding slots
   8.332 +		 */
   8.333 +		for (i = index + nslots - 1; i >= index; i--)
   8.334 +			io_tlb_list[i] = ++count;
   8.335 +		/*
   8.336 +		 * Step 2: merge the returned slots with the preceding slots,
   8.337 +		 * if available (non zero)
   8.338 +		 */
   8.339 +		for (i = index - 1;
   8.340 +		     (OFFSET(i, IO_TLB_SEGSIZE) !=
   8.341 +		      IO_TLB_SEGSIZE -1) && io_tlb_list[i];
   8.342 +		     i--)
   8.343 +			io_tlb_list[i] = ++count;
   8.344 +	}
   8.345 +	spin_unlock_irqrestore(&io_tlb_lock, flags);
   8.346 +}
   8.347 +
   8.348 +static void
   8.349 +sync_single(struct device *hwdev, char *dma_addr, size_t size, int dir)
   8.350 +{
   8.351 +	int index = (dma_addr - io_tlb_start) >> IO_TLB_SHIFT;
   8.352 +	struct phys_addr buffer = io_tlb_orig_addr[index];
   8.353 +	BUG_ON((dir != DMA_FROM_DEVICE) && (dir != DMA_TO_DEVICE));
   8.354 +	__sync_single(buffer, dma_addr, size, dir);
   8.355 +}
   8.356 +
   8.357 +static void
   8.358 +swiotlb_full(struct device *dev, size_t size, int dir, int do_panic)
   8.359 +{
   8.360 +	/*
   8.361 +	 * Ran out of IOMMU space for this operation. This is very bad.
   8.362 +	 * Unfortunately the drivers cannot handle this operation properly.
   8.363 +	 * unless they check for pci_dma_mapping_error (most don't)
   8.364 +	 * When the mapping is small enough return a static buffer to limit
   8.365 +	 * the damage, or panic when the transfer is too big.
   8.366 +	 */
   8.367 +	printk(KERN_ERR "PCI-DMA: Out of SW-IOMMU space for %lu bytes at "
   8.368 +	       "device %s\n", (unsigned long)size, dev ? dev->bus_id : "?");
   8.369 +
   8.370 +	if (size > io_tlb_overflow && do_panic) {
   8.371 +		if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL)
   8.372 +			panic("PCI-DMA: Memory would be corrupted\n");
   8.373 +		if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL)
   8.374 +			panic("PCI-DMA: Random memory would be DMAed\n");
   8.375 +	}
   8.376 +}
   8.377 +
   8.378 +/*
   8.379 + * Map a single buffer of the indicated size for DMA in streaming mode.  The
   8.380 + * PCI address to use is returned.
   8.381 + *
   8.382 + * Once the device is given the dma address, the device owns this memory until
   8.383 + * either swiotlb_unmap_single or swiotlb_dma_sync_single is performed.
   8.384 + */
   8.385 +dma_addr_t
   8.386 +swiotlb_map_single(struct device *hwdev, void *ptr, size_t size, int dir)
   8.387 +{
   8.388 +	dma_addr_t dev_addr = virt_to_bus(ptr);
   8.389 +	void *map;
   8.390 +	struct phys_addr buffer;
   8.391 +
   8.392 +	BUG_ON(dir == DMA_NONE);
   8.393 +
   8.394 +	/*
   8.395 +	 * If the pointer passed in happens to be in the device's DMA window,
   8.396 +	 * we can safely return the device addr and not worry about bounce
   8.397 +	 * buffering it.
   8.398 +	 */
   8.399 +	if (!range_straddles_page_boundary(ptr, size) &&
   8.400 +	    !address_needs_mapping(hwdev, dev_addr))
   8.401 +		return dev_addr;
   8.402 +
   8.403 +	/*
   8.404 +	 * Oh well, have to allocate and map a bounce buffer.
   8.405 +	 */
   8.406 +	buffer.page   = virt_to_page(ptr);
   8.407 +	buffer.offset = (unsigned long)ptr & ~PAGE_MASK;
   8.408 +	map = map_single(hwdev, buffer, size, dir);
   8.409 +	if (!map) {
   8.410 +		swiotlb_full(hwdev, size, dir, 1);
   8.411 +		map = io_tlb_overflow_buffer;
   8.412 +	}
   8.413 +
   8.414 +	dev_addr = virt_to_bus(map);
   8.415 +
   8.416 +	/*
   8.417 +	 * Ensure that the address returned is DMA'ble
   8.418 +	 */
   8.419 +	if (address_needs_mapping(hwdev, dev_addr))
   8.420 +		panic("map_single: bounce buffer is not DMA'ble");
   8.421 +
   8.422 +	return dev_addr;
   8.423 +}
   8.424 +
   8.425 +/*
   8.426 + * Unmap a single streaming mode DMA translation.  The dma_addr and size must
   8.427 + * match what was provided for in a previous swiotlb_map_single call.  All
   8.428 + * other usages are undefined.
   8.429 + *
   8.430 + * After this call, reads by the cpu to the buffer are guaranteed to see
   8.431 + * whatever the device wrote there.
   8.432 + */
   8.433 +void
   8.434 +swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr, size_t size,
   8.435 +		     int dir)
   8.436 +{
   8.437 +	char *dma_addr = bus_to_virt(dev_addr);
   8.438 +
   8.439 +	BUG_ON(dir == DMA_NONE);
   8.440 +	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
   8.441 +		unmap_single(hwdev, dma_addr, size, dir);
   8.442 +}
   8.443 +
   8.444 +/*
   8.445 + * Make physical memory consistent for a single streaming mode DMA translation
   8.446 + * after a transfer.
   8.447 + *
   8.448 + * If you perform a swiotlb_map_single() but wish to interrogate the buffer
   8.449 + * using the cpu, yet do not wish to teardown the PCI dma mapping, you must
   8.450 + * call this function before doing so.  At the next point you give the PCI dma
   8.451 + * address back to the card, you must first perform a
   8.452 + * swiotlb_dma_sync_for_device, and then the device again owns the buffer
   8.453 + */
   8.454 +void
   8.455 +swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
   8.456 +			    size_t size, int dir)
   8.457 +{
   8.458 +	char *dma_addr = bus_to_virt(dev_addr);
   8.459 +
   8.460 +	BUG_ON(dir == DMA_NONE);
   8.461 +	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
   8.462 +		sync_single(hwdev, dma_addr, size, dir);
   8.463 +}
   8.464 +
   8.465 +void
   8.466 +swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
   8.467 +			       size_t size, int dir)
   8.468 +{
   8.469 +	char *dma_addr = bus_to_virt(dev_addr);
   8.470 +
   8.471 +	BUG_ON(dir == DMA_NONE);
   8.472 +	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
   8.473 +		sync_single(hwdev, dma_addr, size, dir);
   8.474 +}
   8.475 +
   8.476 +/*
   8.477 + * Map a set of buffers described by scatterlist in streaming mode for DMA.
   8.478 + * This is the scatter-gather version of the above swiotlb_map_single
   8.479 + * interface.  Here the scatter gather list elements are each tagged with the
   8.480 + * appropriate dma address and length.  They are obtained via
   8.481 + * sg_dma_{address,length}(SG).
   8.482 + *
   8.483 + * NOTE: An implementation may be able to use a smaller number of
   8.484 + *       DMA address/length pairs than there are SG table elements.
   8.485 + *       (for example via virtual mapping capabilities)
   8.486 + *       The routine returns the number of addr/length pairs actually
   8.487 + *       used, at most nents.
   8.488 + *
   8.489 + * Device ownership issues as mentioned above for swiotlb_map_single are the
   8.490 + * same here.
   8.491 + */
   8.492 +int
   8.493 +swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
   8.494 +	       int dir)
   8.495 +{
   8.496 +	struct phys_addr buffer;
   8.497 +	dma_addr_t dev_addr;
   8.498 +	char *map;
   8.499 +	int i;
   8.500 +
   8.501 +	BUG_ON(dir == DMA_NONE);
   8.502 +
   8.503 +	for (i = 0; i < nelems; i++, sg++) {
   8.504 +		dev_addr = SG_ENT_PHYS_ADDRESS(sg);
   8.505 +		if (address_needs_mapping(hwdev, dev_addr)) {
   8.506 +			buffer.page   = sg->page;
   8.507 +			buffer.offset = sg->offset;
   8.508 +			map = map_single(hwdev, buffer, sg->length, dir);
   8.509 +			if (!map) {
   8.510 +				/* Don't panic here, we expect map_sg users
   8.511 +				   to do proper error handling. */
   8.512 +				swiotlb_full(hwdev, sg->length, dir, 0);
   8.513 +				swiotlb_unmap_sg(hwdev, sg - i, i, dir);
   8.514 +				sg[0].dma_length = 0;
   8.515 +				return 0;
   8.516 +			}
   8.517 +			sg->dma_address = (dma_addr_t)virt_to_bus(map);
   8.518 +		} else
   8.519 +			sg->dma_address = dev_addr;
   8.520 +		sg->dma_length = sg->length;
   8.521 +	}
   8.522 +	return nelems;
   8.523 +}
   8.524 +
   8.525 +/*
   8.526 + * Unmap a set of streaming mode DMA translations.  Again, cpu read rules
   8.527 + * concerning calls here are the same as for swiotlb_unmap_single() above.
   8.528 + */
   8.529 +void
   8.530 +swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nelems,
   8.531 +		 int dir)
   8.532 +{
   8.533 +	int i;
   8.534 +
   8.535 +	BUG_ON(dir == DMA_NONE);
   8.536 +
   8.537 +	for (i = 0; i < nelems; i++, sg++)
   8.538 +		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
   8.539 +			unmap_single(hwdev, 
   8.540 +				     (void *)bus_to_virt(sg->dma_address),
   8.541 +				     sg->dma_length, dir);
   8.542 +}
   8.543 +
   8.544 +/*
   8.545 + * Make physical memory consistent for a set of streaming mode DMA translations
   8.546 + * after a transfer.
   8.547 + *
   8.548 + * The same as swiotlb_sync_single_* but for a scatter-gather list, same rules
   8.549 + * and usage.
   8.550 + */
   8.551 +void
   8.552 +swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
   8.553 +			int nelems, int dir)
   8.554 +{
   8.555 +	int i;
   8.556 +
   8.557 +	BUG_ON(dir == DMA_NONE);
   8.558 +
   8.559 +	for (i = 0; i < nelems; i++, sg++)
   8.560 +		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
   8.561 +			sync_single(hwdev,
   8.562 +				    (void *)bus_to_virt(sg->dma_address),
   8.563 +				    sg->dma_length, dir);
   8.564 +}
   8.565 +
   8.566 +void
   8.567 +swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
   8.568 +			   int nelems, int dir)
   8.569 +{
   8.570 +	int i;
   8.571 +
   8.572 +	BUG_ON(dir == DMA_NONE);
   8.573 +
   8.574 +	for (i = 0; i < nelems; i++, sg++)
   8.575 +		if (sg->dma_address != SG_ENT_PHYS_ADDRESS(sg))
   8.576 +			sync_single(hwdev,
   8.577 +				    (void *)bus_to_virt(sg->dma_address),
   8.578 +				    sg->dma_length, dir);
   8.579 +}
   8.580 +
   8.581 +dma_addr_t
   8.582 +swiotlb_map_page(struct device *hwdev, struct page *page,
   8.583 +		 unsigned long offset, size_t size,
   8.584 +		 enum dma_data_direction direction)
   8.585 +{
   8.586 +	struct phys_addr buffer;
   8.587 +	dma_addr_t dev_addr;
   8.588 +	char *map;
   8.589 +
   8.590 +	dev_addr = page_to_phys(page) + offset;
   8.591 +	if (address_needs_mapping(hwdev, dev_addr)) {
   8.592 +		buffer.page   = page;
   8.593 +		buffer.offset = offset;
   8.594 +		map = map_single(hwdev, buffer, size, direction);
   8.595 +		if (!map) {
   8.596 +			swiotlb_full(hwdev, size, direction, 1);
   8.597 +			map = io_tlb_overflow_buffer;
   8.598 +		}
   8.599 +		dev_addr = (dma_addr_t)virt_to_bus(map);
   8.600 +	}
   8.601 +
   8.602 +	return dev_addr;
   8.603 +}
   8.604 +
   8.605 +void
   8.606 +swiotlb_unmap_page(struct device *hwdev, dma_addr_t dma_address,
   8.607 +		   size_t size, enum dma_data_direction direction)
   8.608 +{
   8.609 +	char *dma_addr = bus_to_virt(dma_address);
   8.610 +
   8.611 +	BUG_ON(direction == DMA_NONE);
   8.612 +	if (dma_addr >= io_tlb_start && dma_addr < io_tlb_end)
   8.613 +		unmap_single(hwdev, dma_addr, size, direction);
   8.614 +}
   8.615 +
   8.616 +int
   8.617 +swiotlb_dma_mapping_error(dma_addr_t dma_addr)
   8.618 +{
   8.619 +	return (dma_addr == virt_to_bus(io_tlb_overflow_buffer));
   8.620 +}
   8.621 +
   8.622 +/*
   8.623 + * Return whether the given PCI device DMA address mask can be supported
   8.624 + * properly.  For example, if your device can only drive the low 24-bits
   8.625 + * during PCI bus mastering, then you would pass 0x00ffffff as the mask to
   8.626 + * this function.
   8.627 + */
   8.628 +int
   8.629 +swiotlb_dma_supported (struct device *hwdev, u64 mask)
   8.630 +{
   8.631 +	return (mask >= 0xffffffffUL);
   8.632 +}
   8.633 +
   8.634 +EXPORT_SYMBOL(swiotlb_init);
   8.635 +EXPORT_SYMBOL(swiotlb_map_single);
   8.636 +EXPORT_SYMBOL(swiotlb_unmap_single);
   8.637 +EXPORT_SYMBOL(swiotlb_map_sg);
   8.638 +EXPORT_SYMBOL(swiotlb_unmap_sg);
   8.639 +EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
   8.640 +EXPORT_SYMBOL(swiotlb_sync_single_for_device);
   8.641 +EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
   8.642 +EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
   8.643 +EXPORT_SYMBOL(swiotlb_map_page);
   8.644 +EXPORT_SYMBOL(swiotlb_unmap_page);
   8.645 +EXPORT_SYMBOL(swiotlb_dma_mapping_error);
   8.646 +EXPORT_SYMBOL(swiotlb_dma_supported);
   8.647 +
   8.648 +/*
   8.649 + * Local variables:
   8.650 + *  c-file-style: "linux"
   8.651 + *  indent-tabs-mode: t
   8.652 + *  c-indent-level: 8
   8.653 + *  c-basic-offset: 8
   8.654 + *  tab-width: 8
   8.655 + * End:
   8.656 + */
     9.1 --- a/linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c	Tue Aug 16 10:12:18 2005 +0000
     9.2 +++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/hypervisor.c	Tue Aug 16 11:20:47 2005 +0000
     9.3 @@ -263,12 +263,9 @@ void xen_set_ldt(unsigned long ptr, unsi
     9.4      BUG_ON(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
     9.5  }
     9.6  
     9.7 -void xen_contig_memory(unsigned long vstart, unsigned int order)
     9.8 +/* Ensure multi-page extents are contiguous in machine memory. */
     9.9 +void xen_create_contiguous_region(unsigned long vstart, unsigned int order)
    9.10  {
    9.11 -    /*
    9.12 -     * Ensure multi-page extents are contiguous in machine memory. This code 
    9.13 -     * could be cleaned up some, and the number of hypercalls reduced.
    9.14 -     */
    9.15      pgd_t         *pgd; 
    9.16      pud_t         *pud; 
    9.17      pmd_t         *pmd;
    9.18 @@ -312,6 +309,49 @@ void xen_contig_memory(unsigned long vst
    9.19      balloon_unlock(flags);
    9.20  }
    9.21  
    9.22 +void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order)
    9.23 +{
    9.24 +    pgd_t         *pgd; 
    9.25 +    pud_t         *pud; 
    9.26 +    pmd_t         *pmd;
    9.27 +    pte_t         *pte;
    9.28 +    unsigned long  mfn, i, flags;
    9.29 +
    9.30 +    scrub_pages(vstart, 1 << order);
    9.31 +
    9.32 +    balloon_lock(flags);
    9.33 +
    9.34 +    /* 1. Zap current PTEs, giving away the underlying pages. */
    9.35 +    for (i = 0; i < (1<<order); i++) {
    9.36 +        pgd = pgd_offset_k(vstart + (i*PAGE_SIZE));
    9.37 +        pud = pud_offset(pgd, (vstart + (i*PAGE_SIZE)));
    9.38 +        pmd = pmd_offset(pud, (vstart + (i*PAGE_SIZE)));
    9.39 +        pte = pte_offset_kernel(pmd, (vstart + (i*PAGE_SIZE)));
    9.40 +        mfn = pte_mfn(*pte);
    9.41 +        BUG_ON(HYPERVISOR_update_va_mapping(
    9.42 +            vstart + (i*PAGE_SIZE), __pte_ma(0), 0));
    9.43 +        phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] =
    9.44 +            INVALID_P2M_ENTRY;
    9.45 +        BUG_ON(HYPERVISOR_dom_mem_op(
    9.46 +            MEMOP_decrease_reservation, &mfn, 1, 0) != 1);
    9.47 +    }
    9.48 +
    9.49 +    /* 2. Map new pages in place of old pages. */
    9.50 +    for (i = 0; i < (1<<order); i++) {
    9.51 +        BUG_ON(HYPERVISOR_dom_mem_op(
    9.52 +            MEMOP_increase_reservation, &mfn, 1, 0) != 1);
    9.53 +        BUG_ON(HYPERVISOR_update_va_mapping(
    9.54 +            vstart + (i*PAGE_SIZE),
    9.55 +            __pte_ma((mfn<<PAGE_SHIFT)|__PAGE_KERNEL), 0));
    9.56 +        xen_machphys_update(mfn, (__pa(vstart)>>PAGE_SHIFT)+i);
    9.57 +        phys_to_machine_mapping[(__pa(vstart)>>PAGE_SHIFT)+i] = mfn;
    9.58 +    }
    9.59 +
    9.60 +    flush_tlb_all();
    9.61 +
    9.62 +    balloon_unlock(flags);
    9.63 +}
    9.64 +
    9.65  #ifdef CONFIG_XEN_PHYSDEV_ACCESS
    9.66  
    9.67  unsigned long allocate_empty_lowmem_region(unsigned long pages)
    10.1 --- a/linux-2.6-xen-sparse/arch/xen/i386/mm/init.c	Tue Aug 16 10:12:18 2005 +0000
    10.2 +++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/init.c	Tue Aug 16 11:20:47 2005 +0000
    10.3 @@ -41,6 +41,12 @@
    10.4  #include <asm/sections.h>
    10.5  #include <asm-xen/hypervisor.h>
    10.6  
    10.7 +#if defined(CONFIG_SWIOTLB)
    10.8 +extern void swiotlb_init(void);
    10.9 +int swiotlb;
   10.10 +EXPORT_SYMBOL(swiotlb);
   10.11 +#endif
   10.12 +
   10.13  unsigned int __VMALLOC_RESERVE = 128 << 20;
   10.14  
   10.15  DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
   10.16 @@ -631,6 +637,10 @@ void __init mem_init(void)
   10.17  	int bad_ppro;
   10.18  	unsigned long pfn;
   10.19  
   10.20 +#if defined(CONFIG_SWIOTLB)
   10.21 +	swiotlb_init();	
   10.22 +#endif
   10.23 +
   10.24  #ifndef CONFIG_DISCONTIGMEM
   10.25  	if (!mem_map)
   10.26  		BUG();
    11.1 --- a/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c	Tue Aug 16 10:12:18 2005 +0000
    11.2 +++ b/linux-2.6-xen-sparse/arch/xen/i386/mm/pgtable.c	Tue Aug 16 11:20:47 2005 +0000
    11.3 @@ -277,7 +277,7 @@ void pgd_ctor(void *pgd, kmem_cache_t *c
    11.4  
    11.5  #ifdef CONFIG_X86_PAE
    11.6  	/* this gives us a page below 4GB */
    11.7 -	xen_contig_memory((unsigned long)pgd, 0);
    11.8 +	xen_create_contiguous_region((unsigned long)pgd, 0);
    11.9  #endif
   11.10  
   11.11  	if (!HAVE_SHARED_KERNEL_PMD)
    12.1 --- a/linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig	Tue Aug 16 10:12:18 2005 +0000
    12.2 +++ b/linux-2.6-xen-sparse/arch/xen/x86_64/Kconfig	Tue Aug 16 11:20:47 2005 +0000
    12.3 @@ -329,12 +329,12 @@ config GART_IOMMU
    12.4  # need this always enabled with GART_IOMMU for the VIA workaround
    12.5  config SWIOTLB
    12.6         bool
    12.7 -       depends on GART_IOMMU
    12.8 +       depends on PCI
    12.9         default y
   12.10  
   12.11  config DUMMY_IOMMU
   12.12  	bool
   12.13 -	depends on !GART_IOMMU && !SWIOTLB
   12.14 +	depends on !GART_IOMMU
   12.15  	default y
   12.16  	help
   12.17  	  Don't use IOMMU code. This will cause problems when you have more than 4GB
    13.1 --- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile	Tue Aug 16 10:12:18 2005 +0000
    13.2 +++ b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/Makefile	Tue Aug 16 11:20:47 2005 +0000
    13.3 @@ -36,8 +36,9 @@ c-obj-$(CONFIG_X86_IO_APIC)	+= genapic_c
    13.4  #obj-$(CONFIG_CPU_FREQ)		+= cpufreq/
    13.5  #obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
    13.6  #obj-$(CONFIG_GART_IOMMU)	+= pci-gart.o aperture.o
    13.7 -obj-$(CONFIG_DUMMY_IOMMU)	+= pci-nommu.o pci-dma.o
    13.8 -#obj-$(CONFIG_SWIOTLB)		+= swiotlb.o
    13.9 +obj-$(CONFIG_DUMMY_IOMMU)	+= pci-nommu.o
   13.10 +i386-obj-$(CONFIG_DUMMY_IOMMU)	+= pci-dma.o
   13.11 +i386-obj-$(CONFIG_SWIOTLB)	+= swiotlb.o
   13.12  obj-$(CONFIG_KPROBES)		+= kprobes.o
   13.13  obj-$(CONFIG_X86_PM_TIMER)	+= pmtimer.o
   13.14  
   13.15 @@ -49,7 +50,7 @@ c-obj-y				+= intel_cacheinfo.o
   13.16  bootflag-y			+= ../../../i386/kernel/bootflag.o
   13.17  cpuid-$(subst m,y,$(CONFIG_X86_CPUID))  += ../../../i386/kernel/cpuid.o
   13.18  topology-y                     += ../../../i386/mach-default/topology.o
   13.19 -swiotlb-$(CONFIG_SWIOTLB)      += ../../../ia64/lib/swiotlb.o
   13.20 +#swiotlb-$(CONFIG_SWIOTLB)      += ../../../ia64/lib/swiotlb.o
   13.21  microcode-$(subst m,y,$(CONFIG_MICROCODE))  += ../../../i386/kernel/microcode.o
   13.22  intel_cacheinfo-y		+= ../../../i386/kernel/cpu/intel_cacheinfo.o
   13.23  quirks-y			+= ../../i386/kernel/quirks.o
    14.1 --- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c	Tue Aug 16 10:12:18 2005 +0000
    14.2 +++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
    14.3 @@ -1,336 +0,0 @@
    14.4 -/*
    14.5 - * Dynamic DMA mapping support.
    14.6 - */
    14.7 -
    14.8 -#include <linux/types.h>
    14.9 -#include <linux/mm.h>
   14.10 -#include <linux/string.h>
   14.11 -#include <linux/pci.h>
   14.12 -#include <linux/module.h>
   14.13 -#include <asm/io.h>
   14.14 -#include <asm-xen/balloon.h>
   14.15 -
   14.16 -/* Map a set of buffers described by scatterlist in streaming
   14.17 - * mode for DMA.  This is the scatter-gather version of the
   14.18 - * above pci_map_single interface.  Here the scatter gather list
   14.19 - * elements are each tagged with the appropriate dma address
   14.20 - * and length.  They are obtained via sg_dma_{address,length}(SG).
   14.21 - *
   14.22 - * NOTE: An implementation may be able to use a smaller number of
   14.23 - *       DMA address/length pairs than there are SG table elements.
   14.24 - *       (for example via virtual mapping capabilities)
   14.25 - *       The routine returns the number of addr/length pairs actually
   14.26 - *       used, at most nents.
   14.27 - *
   14.28 - * Device ownership issues as mentioned above for pci_map_single are
   14.29 - * the same here.
   14.30 - */
   14.31 -int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
   14.32 -	       int nents, int direction)
   14.33 -{
   14.34 -	int i;
   14.35 -
   14.36 -	BUG_ON(direction == DMA_NONE);
   14.37 - 	for (i = 0; i < nents; i++ ) {
   14.38 -		struct scatterlist *s = &sg[i];
   14.39 -		BUG_ON(!s->page); 
   14.40 -		s->dma_address = virt_to_bus(page_address(s->page) +s->offset);
   14.41 -		s->dma_length = s->length;
   14.42 -	}
   14.43 -	return nents;
   14.44 -}
   14.45 -
   14.46 -EXPORT_SYMBOL(dma_map_sg);
   14.47 -
   14.48 -/* Unmap a set of streaming mode DMA translations.
   14.49 - * Again, cpu read rules concerning calls here are the same as for
   14.50 - * pci_unmap_single() above.
   14.51 - */
   14.52 -void dma_unmap_sg(struct device *dev, struct scatterlist *sg,
   14.53 -		  int nents, int dir)
   14.54 -{
   14.55 -	int i;
   14.56 -	for (i = 0; i < nents; i++) { 
   14.57 -		struct scatterlist *s = &sg[i];
   14.58 -		BUG_ON(s->page == NULL); 
   14.59 -		BUG_ON(s->dma_address == 0); 
   14.60 -		dma_unmap_single(dev, s->dma_address, s->dma_length, dir);
   14.61 -	} 
   14.62 -}
   14.63 -
   14.64 -EXPORT_SYMBOL(dma_unmap_sg);
   14.65 -
   14.66 -struct dma_coherent_mem {
   14.67 -	void		*virt_base;
   14.68 -	u32		device_base;
   14.69 -	int		size;
   14.70 -	int		flags;
   14.71 -	unsigned long	*bitmap;
   14.72 -};
   14.73 -
   14.74 -void *dma_alloc_coherent(struct device *dev, size_t size,
   14.75 -			   dma_addr_t *dma_handle, unsigned gfp)
   14.76 -{
   14.77 -	void *ret;
   14.78 -	unsigned int order = get_order(size);
   14.79 -	unsigned long vstart;
   14.80 -
   14.81 -	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
   14.82 -
   14.83 -	/* ignore region specifiers */
   14.84 -	gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
   14.85 -
   14.86 -	if (mem) {
   14.87 -		int page = bitmap_find_free_region(mem->bitmap, mem->size,
   14.88 -						     order);
   14.89 -		if (page >= 0) {
   14.90 -			*dma_handle = mem->device_base + (page << PAGE_SHIFT);
   14.91 -			ret = mem->virt_base + (page << PAGE_SHIFT);
   14.92 -			memset(ret, 0, size);
   14.93 -			return ret;
   14.94 -		}
   14.95 -		if (mem->flags & DMA_MEMORY_EXCLUSIVE)
   14.96 -			return NULL;
   14.97 -	}
   14.98 -
   14.99 -	if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
  14.100 -		gfp |= GFP_DMA;
  14.101 -
  14.102 -	vstart = __get_free_pages(gfp, order);
  14.103 -	ret = (void *)vstart;
  14.104 -	if (ret == NULL)
  14.105 -		return ret;
  14.106 -
  14.107 -	xen_contig_memory(vstart, order);
  14.108 -
  14.109 -	memset(ret, 0, size);
  14.110 -	*dma_handle = virt_to_bus(ret);
  14.111 -
  14.112 -	return ret;
  14.113 -}
  14.114 -EXPORT_SYMBOL(dma_alloc_coherent);
  14.115 -
  14.116 -void dma_free_coherent(struct device *dev, size_t size,
  14.117 -			 void *vaddr, dma_addr_t dma_handle)
  14.118 -{
  14.119 -	struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
  14.120 -	int order = get_order(size);
  14.121 -	
  14.122 -	if (mem && vaddr >= mem->virt_base && vaddr < (mem->virt_base + (mem->size << PAGE_SHIFT))) {
  14.123 -		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
  14.124 -
  14.125 -		bitmap_release_region(mem->bitmap, page, order);
  14.126 -	} else
  14.127 -		free_pages((unsigned long)vaddr, order);
  14.128 -}
  14.129 -EXPORT_SYMBOL(dma_free_coherent);
  14.130 -
  14.131 -#if 0
  14.132 -int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
  14.133 -				dma_addr_t device_addr, size_t size, int flags)
  14.134 -{
  14.135 -	void __iomem *mem_base;
  14.136 -	int pages = size >> PAGE_SHIFT;
  14.137 -	int bitmap_size = (pages + 31)/32;
  14.138 -
  14.139 -	if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
  14.140 -		goto out;
  14.141 -	if (!size)
  14.142 -		goto out;
  14.143 -	if (dev->dma_mem)
  14.144 -		goto out;
  14.145 -
  14.146 -	/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
  14.147 -
  14.148 -	mem_base = ioremap(bus_addr, size);
  14.149 -	if (!mem_base)
  14.150 -		goto out;
  14.151 -
  14.152 -	dev->dma_mem = kmalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
  14.153 -	if (!dev->dma_mem)
  14.154 -		goto out;
  14.155 -	memset(dev->dma_mem, 0, sizeof(struct dma_coherent_mem));
  14.156 -	dev->dma_mem->bitmap = kmalloc(bitmap_size, GFP_KERNEL);
  14.157 -	if (!dev->dma_mem->bitmap)
  14.158 -		goto free1_out;
  14.159 -	memset(dev->dma_mem->bitmap, 0, bitmap_size);
  14.160 -
  14.161 -	dev->dma_mem->virt_base = mem_base;
  14.162 -	dev->dma_mem->device_base = device_addr;
  14.163 -	dev->dma_mem->size = pages;
  14.164 -	dev->dma_mem->flags = flags;
  14.165 -
  14.166 -	if (flags & DMA_MEMORY_MAP)
  14.167 -		return DMA_MEMORY_MAP;
  14.168 -
  14.169 -	return DMA_MEMORY_IO;
  14.170 -
  14.171 - free1_out:
  14.172 -	kfree(dev->dma_mem->bitmap);
  14.173 - out:
  14.174 -	return 0;
  14.175 -}
  14.176 -EXPORT_SYMBOL(dma_declare_coherent_memory);
  14.177 -
  14.178 -void dma_release_declared_memory(struct device *dev)
  14.179 -{
  14.180 -	struct dma_coherent_mem *mem = dev->dma_mem;
  14.181 -	
  14.182 -	if(!mem)
  14.183 -		return;
  14.184 -	dev->dma_mem = NULL;
  14.185 -	iounmap(mem->virt_base);
  14.186 -	kfree(mem->bitmap);
  14.187 -	kfree(mem);
  14.188 -}
  14.189 -EXPORT_SYMBOL(dma_release_declared_memory);
  14.190 -
  14.191 -void *dma_mark_declared_memory_occupied(struct device *dev,
  14.192 -					dma_addr_t device_addr, size_t size)
  14.193 -{
  14.194 -	struct dma_coherent_mem *mem = dev->dma_mem;
  14.195 -	int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT;
  14.196 -	int pos, err;
  14.197 -
  14.198 -	if (!mem)
  14.199 -		return ERR_PTR(-EINVAL);
  14.200 -
  14.201 -	pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
  14.202 -	err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
  14.203 -	if (err != 0)
  14.204 -		return ERR_PTR(err);
  14.205 -	return mem->virt_base + (pos << PAGE_SHIFT);
  14.206 -}
  14.207 -EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
  14.208 -#endif
  14.209 -
  14.210 -static LIST_HEAD(dma_map_head);
  14.211 -static DEFINE_SPINLOCK(dma_map_lock);
  14.212 -struct dma_map_entry {
  14.213 -	struct list_head list;
  14.214 -	dma_addr_t dma;
  14.215 -	char *bounce, *host;
  14.216 -	size_t size;
  14.217 -};
  14.218 -#define DMA_MAP_MATCHES(e,d) (((e)->dma<=(d)) && (((e)->dma+(e)->size)>(d)))
  14.219 -
  14.220 -dma_addr_t
  14.221 -dma_map_single(struct device *dev, void *ptr, size_t size,
  14.222 -	       enum dma_data_direction direction)
  14.223 -{
  14.224 -	struct dma_map_entry *ent;
  14.225 -	void *bnc;
  14.226 -	dma_addr_t dma;
  14.227 -	unsigned long flags;
  14.228 -
  14.229 -	if (direction == DMA_NONE)
  14.230 -		out_of_line_bug();
  14.231 -
  14.232 -	/*
  14.233 -	 * Even if size is sub-page, the buffer may still straddle a page
  14.234 -	 * boundary. Take into account buffer start offset. All other calls are
  14.235 -	 * conservative and always search the dma_map list if it's non-empty.
  14.236 -	 */
  14.237 -	if (((((unsigned long)ptr) & ~PAGE_MASK) + size) <= PAGE_SIZE) {
  14.238 -		dma = virt_to_bus(ptr);
  14.239 -	} else {
  14.240 -		BUG_ON((bnc = dma_alloc_coherent(dev, size, &dma, GFP_ATOMIC)) == NULL);
  14.241 -		BUG_ON((ent = kmalloc(sizeof(*ent), GFP_ATOMIC)) == NULL);
  14.242 -		if (direction != DMA_FROM_DEVICE)
  14.243 -			memcpy(bnc, ptr, size);
  14.244 -		ent->dma    = dma;
  14.245 -		ent->bounce = bnc;
  14.246 -		ent->host   = ptr;
  14.247 -		ent->size   = size;
  14.248 -		spin_lock_irqsave(&dma_map_lock, flags);
  14.249 -		list_add(&ent->list, &dma_map_head);
  14.250 -		spin_unlock_irqrestore(&dma_map_lock, flags);
  14.251 -	}
  14.252 -
  14.253 -	if ((dma+size) & ~*dev->dma_mask)
  14.254 -		out_of_line_bug();
  14.255 -	return dma;
  14.256 -}
  14.257 -EXPORT_SYMBOL(dma_map_single);
  14.258 -
  14.259 -void
  14.260 -dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
  14.261 -		 enum dma_data_direction direction)
  14.262 -{
  14.263 -	struct dma_map_entry *ent;
  14.264 -	unsigned long flags;
  14.265 -
  14.266 -	if (direction == DMA_NONE)
  14.267 -		out_of_line_bug();
  14.268 -
  14.269 -	/* Fast-path check: are there any multi-page DMA mappings? */
  14.270 -	if (!list_empty(&dma_map_head)) {
  14.271 -		spin_lock_irqsave(&dma_map_lock, flags);
  14.272 -		list_for_each_entry ( ent, &dma_map_head, list ) {
  14.273 -			if (DMA_MAP_MATCHES(ent, dma_addr)) {
  14.274 -				list_del(&ent->list);
  14.275 -				break;
  14.276 -			}
  14.277 -		}
  14.278 -		spin_unlock_irqrestore(&dma_map_lock, flags);
  14.279 -		if (&ent->list != &dma_map_head) {
  14.280 -			BUG_ON(dma_addr != ent->dma);
  14.281 -			BUG_ON(size != ent->size);
  14.282 -			if (direction != DMA_TO_DEVICE)
  14.283 -				memcpy(ent->host, ent->bounce, size);
  14.284 -			dma_free_coherent(dev, size, ent->bounce, ent->dma);
  14.285 -			kfree(ent);
  14.286 -		}
  14.287 -	}
  14.288 -}
  14.289 -EXPORT_SYMBOL(dma_unmap_single);
  14.290 -
  14.291 -void
  14.292 -dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
  14.293 -			enum dma_data_direction direction)
  14.294 -{
  14.295 -	struct dma_map_entry *ent;
  14.296 -	unsigned long flags, off;
  14.297 -
  14.298 -	/* Fast-path check: are there any multi-page DMA mappings? */
  14.299 -	if (!list_empty(&dma_map_head)) {
  14.300 -		spin_lock_irqsave(&dma_map_lock, flags);
  14.301 -		list_for_each_entry ( ent, &dma_map_head, list )
  14.302 -			if (DMA_MAP_MATCHES(ent, dma_handle))
  14.303 -				break;
  14.304 -		spin_unlock_irqrestore(&dma_map_lock, flags);
  14.305 -		if (&ent->list != &dma_map_head) {
  14.306 -			off = dma_handle - ent->dma;
  14.307 -			BUG_ON((off + size) > ent->size);
  14.308 -			/*if (direction != DMA_TO_DEVICE)*/
  14.309 -				memcpy(ent->host+off, ent->bounce+off, size);
  14.310 -		}
  14.311 -	}
  14.312 -}
  14.313 -EXPORT_SYMBOL(dma_sync_single_for_cpu);
  14.314 -
  14.315 -void
  14.316 -dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size,
  14.317 -                           enum dma_data_direction direction)
  14.318 -{
  14.319 -	struct dma_map_entry *ent;
  14.320 -	unsigned long flags, off;
  14.321 -
  14.322 -	/* Fast-path check: are there any multi-page DMA mappings? */
  14.323 -	if (!list_empty(&dma_map_head)) {
  14.324 -		spin_lock_irqsave(&dma_map_lock, flags);
  14.325 -		list_for_each_entry ( ent, &dma_map_head, list )
  14.326 -			if (DMA_MAP_MATCHES(ent, dma_handle))
  14.327 -				break;
  14.328 -		spin_unlock_irqrestore(&dma_map_lock, flags);
  14.329 -		if (&ent->list != &dma_map_head) {
  14.330 -			off = dma_handle - ent->dma;
  14.331 -			BUG_ON((off + size) > ent->size);
  14.332 -			/*if (direction != DMA_FROM_DEVICE)*/
  14.333 -				memcpy(ent->bounce+off, ent->host+off, size);
  14.334 -		}
  14.335 -	}
  14.336 -
  14.337 -	flush_write_buffers();
  14.338 -}
  14.339 -EXPORT_SYMBOL(dma_sync_single_for_device);
    15.1 --- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c	Tue Aug 16 10:12:18 2005 +0000
    15.2 +++ b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-nommu.c	Tue Aug 16 11:20:47 2005 +0000
    15.3 @@ -61,6 +61,7 @@ void dma_free_coherent(struct device *hw
    15.4  EXPORT_SYMBOL(dma_free_coherent);
    15.5  #endif
    15.6  
    15.7 +#if 0
    15.8  int dma_supported(struct device *hwdev, u64 mask)
    15.9  {
   15.10          /*
   15.11 @@ -76,6 +77,7 @@ int dma_supported(struct device *hwdev, 
   15.12  	return 1;
   15.13  } 
   15.14  EXPORT_SYMBOL(dma_supported);
   15.15 +#endif
   15.16  
   15.17  int dma_get_cache_alignment(void)
   15.18  {
    16.1 --- a/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c	Tue Aug 16 10:12:18 2005 +0000
    16.2 +++ b/linux-2.6-xen-sparse/arch/xen/x86_64/mm/init.c	Tue Aug 16 11:20:47 2005 +0000
    16.3 @@ -44,10 +44,6 @@
    16.4  #define Dprintk(x...)
    16.5  #endif
    16.6  
    16.7 -#ifdef CONFIG_GART_IOMMU
    16.8 -extern int swiotlb;
    16.9 -#endif
   16.10 -
   16.11  extern char _stext[];
   16.12  
   16.13  DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
   16.14 @@ -790,8 +786,6 @@ static inline int page_is_ram (unsigned 
   16.15          return 1;
   16.16  }
   16.17  
   16.18 -extern int swiotlb_force;
   16.19 -
   16.20  static struct kcore_list kcore_mem, kcore_vmalloc, kcore_kernel, kcore_modules,
   16.21  			 kcore_vsyscall;
   16.22  
   16.23 @@ -800,14 +794,9 @@ void __init mem_init(void)
   16.24  	int codesize, reservedpages, datasize, initsize;
   16.25  	int tmp;
   16.26  
   16.27 -#ifdef CONFIG_SWIOTLB
   16.28 -	if (swiotlb_force)
   16.29 -		swiotlb = 1;
   16.30 -	if (!iommu_aperture &&
   16.31 -	    (end_pfn >= 0xffffffff>>PAGE_SHIFT || force_iommu))
   16.32 -	       swiotlb = 1;
   16.33 -	if (swiotlb)
   16.34 -		swiotlb_init();	
   16.35 +#if defined(CONFIG_SWIOTLB)
   16.36 +	extern void swiotlb_init(void);
   16.37 +	swiotlb_init();	
   16.38  #endif
   16.39  
   16.40  	/* How many end-of-memory variables you have, grandma! */
    17.1 --- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h	Tue Aug 16 10:12:18 2005 +0000
    17.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h	Tue Aug 16 11:20:47 2005 +0000
    17.3 @@ -1,11 +1,33 @@
    17.4  #ifndef _ASM_I386_DMA_MAPPING_H
    17.5  #define _ASM_I386_DMA_MAPPING_H
    17.6  
    17.7 +/*
    17.8 + * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for
    17.9 + * documentation.
   17.10 + */
   17.11 +
   17.12 +#include <linux/config.h>
   17.13  #include <linux/mm.h>
   17.14 -
   17.15  #include <asm/cache.h>
   17.16  #include <asm/io.h>
   17.17  #include <asm/scatterlist.h>
   17.18 +#include <asm-i386/swiotlb.h>
   17.19 +
   17.20 +static inline int
   17.21 +address_needs_mapping(struct device *hwdev, dma_addr_t addr)
   17.22 +{
   17.23 +	dma_addr_t mask = 0xffffffff;
   17.24 +	/* If the device has a mask, use it, otherwise default to 32 bits */
   17.25 +	if (hwdev && hwdev->dma_mask)
   17.26 +		mask = *hwdev->dma_mask;
   17.27 +	return (addr & ~mask) != 0;
   17.28 +}
   17.29 +
   17.30 +static inline int
   17.31 +range_straddles_page_boundary(void *p, size_t size)
   17.32 +{
   17.33 +	return ((((unsigned long)p & ~PAGE_MASK) + size) > PAGE_SIZE);
   17.34 +}
   17.35  
   17.36  #define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f)
   17.37  #define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h)
   17.38 @@ -24,46 +46,18 @@ extern void
   17.39  dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
   17.40  		 enum dma_data_direction direction);
   17.41  
   17.42 -static inline int
   17.43 -dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
   17.44 -	   enum dma_data_direction direction)
   17.45 -{
   17.46 -	int i;
   17.47 -
   17.48 -	BUG_ON(direction == DMA_NONE);
   17.49 +extern int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
   17.50 +		      int nents, enum dma_data_direction direction);
   17.51 +extern void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg,
   17.52 +			 int nents, enum dma_data_direction direction);
   17.53  
   17.54 -	for (i = 0; i < nents; i++ ) {
   17.55 -		BUG_ON(!sg[i].page);
   17.56 -
   17.57 -		sg[i].dma_address = page_to_phys(sg[i].page) + sg[i].offset;
   17.58 -	}
   17.59 -
   17.60 -	flush_write_buffers();
   17.61 -	return nents;
   17.62 -}
   17.63 -
   17.64 -static inline dma_addr_t
   17.65 +extern dma_addr_t
   17.66  dma_map_page(struct device *dev, struct page *page, unsigned long offset,
   17.67 -	     size_t size, enum dma_data_direction direction)
   17.68 -{
   17.69 -	BUG_ON(direction == DMA_NONE);
   17.70 -	return page_to_phys(page) + offset;
   17.71 -}
   17.72 +	     size_t size, enum dma_data_direction direction);
   17.73  
   17.74 -static inline void
   17.75 +extern void
   17.76  dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
   17.77 -	       enum dma_data_direction direction)
   17.78 -{
   17.79 -	BUG_ON(direction == DMA_NONE);
   17.80 -}
   17.81 -
   17.82 -
   17.83 -static inline void
   17.84 -dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
   17.85 -	     enum dma_data_direction direction)
   17.86 -{
   17.87 -	BUG_ON(direction == DMA_NONE);
   17.88 -}
   17.89 +	       enum dma_data_direction direction);
   17.90  
   17.91  extern void
   17.92  dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
   17.93 @@ -93,34 +87,25 @@ static inline void
   17.94  dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
   17.95  		    enum dma_data_direction direction)
   17.96  {
   17.97 +	if (swiotlb)
   17.98 +		swiotlb_sync_sg_for_cpu(dev,sg,nelems,direction);
   17.99 +	flush_write_buffers();
  17.100  }
  17.101  
  17.102  static inline void
  17.103  dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
  17.104  		    enum dma_data_direction direction)
  17.105  {
  17.106 +	if (swiotlb)
  17.107 +		swiotlb_sync_sg_for_device(dev,sg,nelems,direction);
  17.108  	flush_write_buffers();
  17.109  }
  17.110  
  17.111 -static inline int
  17.112 -dma_mapping_error(dma_addr_t dma_addr)
  17.113 -{
  17.114 -	return 0;
  17.115 -}
  17.116 +extern int
  17.117 +dma_mapping_error(dma_addr_t dma_addr);
  17.118  
  17.119 -static inline int
  17.120 -dma_supported(struct device *dev, u64 mask)
  17.121 -{
  17.122 -        /*
  17.123 -         * we fall back to GFP_DMA when the mask isn't all 1s,
  17.124 -         * so we can't guarantee allocations that must be
  17.125 -         * within a tighter range than GFP_DMA..
  17.126 -         */
  17.127 -        if(mask < 0x00ffffff)
  17.128 -                return 0;
  17.129 -
  17.130 -	return 1;
  17.131 -}
  17.132 +extern int
  17.133 +dma_supported(struct device *dev, u64 mask);
  17.134  
  17.135  static inline int
  17.136  dma_set_mask(struct device *dev, u64 mask)
  17.137 @@ -133,6 +118,7 @@ dma_set_mask(struct device *dev, u64 mas
  17.138  	return 0;
  17.139  }
  17.140  
  17.141 +#ifdef __i386__
  17.142  static inline int
  17.143  dma_get_cache_alignment(void)
  17.144  {
  17.145 @@ -140,6 +126,9 @@ dma_get_cache_alignment(void)
  17.146  	 * maximum possible, to be safe */
  17.147  	return (1 << L1_CACHE_SHIFT_MAX);
  17.148  }
  17.149 +#else
  17.150 +extern int dma_get_cache_alignment(void);
  17.151 +#endif
  17.152  
  17.153  #define dma_is_consistent(d)	(1)
  17.154  
    18.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    18.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/kmap_types.h	Tue Aug 16 11:20:47 2005 +0000
    18.3 @@ -0,0 +1,32 @@
    18.4 +#ifndef _ASM_KMAP_TYPES_H
    18.5 +#define _ASM_KMAP_TYPES_H
    18.6 +
    18.7 +#include <linux/config.h>
    18.8 +
    18.9 +#ifdef CONFIG_DEBUG_HIGHMEM
   18.10 +# define D(n) __KM_FENCE_##n ,
   18.11 +#else
   18.12 +# define D(n)
   18.13 +#endif
   18.14 +
   18.15 +enum km_type {
   18.16 +D(0)	KM_BOUNCE_READ,
   18.17 +D(1)	KM_SKB_SUNRPC_DATA,
   18.18 +D(2)	KM_SKB_DATA_SOFTIRQ,
   18.19 +D(3)	KM_USER0,
   18.20 +D(4)	KM_USER1,
   18.21 +D(5)	KM_BIO_SRC_IRQ,
   18.22 +D(6)	KM_BIO_DST_IRQ,
   18.23 +D(7)	KM_PTE0,
   18.24 +D(8)	KM_PTE1,
   18.25 +D(9)	KM_IRQ0,
   18.26 +D(10)	KM_IRQ1,
   18.27 +D(11)	KM_SOFTIRQ0,
   18.28 +D(12)	KM_SOFTIRQ1,
   18.29 +D(13)	KM_SWIOTLB,
   18.30 +D(14)	KM_TYPE_NR
   18.31 +};
   18.32 +
   18.33 +#undef D
   18.34 +
   18.35 +#endif
    19.1 --- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h	Tue Aug 16 10:12:18 2005 +0000
    19.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/pci.h	Tue Aug 16 11:20:47 2005 +0000
    19.3 @@ -43,11 +43,8 @@ int pcibios_set_irq_routing(struct pci_d
    19.4  
    19.5  struct pci_dev;
    19.6  
    19.7 -/* The PCI address space does equal the physical memory
    19.8 - * address space.  The networking and block device layers use
    19.9 - * this boolean for bounce buffer decisions.
   19.10 - */
   19.11 -#define PCI_DMA_BUS_IS_PHYS	(1)
   19.12 +/* On Xen we use SWIOTLB instead of blk-specific bounce buffers. */
   19.13 +#define PCI_DMA_BUS_IS_PHYS	(0)
   19.14  
   19.15  /* pci_unmap_{page,single} is a nop so... */
   19.16  #define DECLARE_PCI_UNMAP_ADDR(ADDR_NAME)
    20.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    20.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/scatterlist.h	Tue Aug 16 11:20:47 2005 +0000
    20.3 @@ -0,0 +1,22 @@
    20.4 +#ifndef _I386_SCATTERLIST_H
    20.5 +#define _I386_SCATTERLIST_H
    20.6 +
    20.7 +struct scatterlist {
    20.8 +    struct page		*page;
    20.9 +    unsigned int	offset;
   20.10 +    unsigned int	length;
   20.11 +    dma_addr_t		dma_address;
   20.12 +    unsigned int	dma_length;
   20.13 +};
   20.14 +
   20.15 +/* These macros should be used after a pci_map_sg call has been done
   20.16 + * to get bus addresses of each of the SG entries and their lengths.
   20.17 + * You should only work with the number of sg entries pci_map_sg
   20.18 + * returns.
   20.19 + */
   20.20 +#define sg_dma_address(sg)	((sg)->dma_address)
   20.21 +#define sg_dma_len(sg)		((sg)->dma_length)
   20.22 +
   20.23 +#define ISA_DMA_THRESHOLD (0x00ffffff)
   20.24 +
   20.25 +#endif /* !(_I386_SCATTERLIST_H) */
    21.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
    21.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/swiotlb.h	Tue Aug 16 11:20:47 2005 +0000
    21.3 @@ -0,0 +1,42 @@
    21.4 +#ifndef _ASM_SWIOTLB_H
    21.5 +#define _ASM_SWIOTLB_H 1
    21.6 +
    21.7 +#include <linux/config.h>
    21.8 +
    21.9 +/* SWIOTLB interface */
   21.10 +
   21.11 +extern dma_addr_t swiotlb_map_single(struct device *hwdev, void *ptr, size_t size,
   21.12 +				      int dir);
   21.13 +extern void swiotlb_unmap_single(struct device *hwdev, dma_addr_t dev_addr,
   21.14 +				  size_t size, int dir);
   21.15 +extern void swiotlb_sync_single_for_cpu(struct device *hwdev,
   21.16 +					 dma_addr_t dev_addr,
   21.17 +					 size_t size, int dir);
   21.18 +extern void swiotlb_sync_single_for_device(struct device *hwdev,
   21.19 +					    dma_addr_t dev_addr,
   21.20 +					    size_t size, int dir);
   21.21 +extern void swiotlb_sync_sg_for_cpu(struct device *hwdev,
   21.22 +				     struct scatterlist *sg, int nelems,
   21.23 +				     int dir);
   21.24 +extern void swiotlb_sync_sg_for_device(struct device *hwdev,
   21.25 +					struct scatterlist *sg, int nelems,
   21.26 +					int dir);
   21.27 +extern int swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg,
   21.28 +		      int nents, int direction);
   21.29 +extern void swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg,
   21.30 +			 int nents, int direction);
   21.31 +extern int swiotlb_dma_mapping_error(dma_addr_t dma_addr);
   21.32 +extern dma_addr_t swiotlb_map_page(struct device *hwdev, struct page *page,
   21.33 +                                   unsigned long offset, size_t size,
   21.34 +                                   enum dma_data_direction direction);
   21.35 +extern void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dma_address,
   21.36 +                               size_t size, enum dma_data_direction direction);
   21.37 +extern int swiotlb_dma_supported(struct device *hwdev, u64 mask);
   21.38 +
   21.39 +#ifdef CONFIG_SWIOTLB
   21.40 +extern int swiotlb;
   21.41 +#else
   21.42 +#define swiotlb 0
   21.43 +#endif
   21.44 +
   21.45 +#endif
    22.1 --- a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h	Tue Aug 16 10:12:18 2005 +0000
    22.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h	Tue Aug 16 11:20:47 2005 +0000
    22.3 @@ -1,89 +1,1 @@
    22.4 -#ifndef _X8664_DMA_MAPPING_H
    22.5 -#define _X8664_DMA_MAPPING_H 1
    22.6 -
    22.7 -/*
    22.8 - * IOMMU interface. See Documentation/DMA-mapping.txt and DMA-API.txt for
    22.9 - * documentation.
   22.10 - */
   22.11 -
   22.12 -#include <linux/config.h>
   22.13 -
   22.14 -#include <asm/scatterlist.h>
   22.15 -#include <asm/io.h>
   22.16 -#include <asm/swiotlb.h>
   22.17 -
   22.18 -extern dma_addr_t bad_dma_address;
   22.19 -#define dma_mapping_error(x) \
   22.20 -	(swiotlb ? swiotlb_dma_mapping_error(x) : ((x) == bad_dma_address))
   22.21 -
   22.22 -void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
   22.23 -			 unsigned gfp);
   22.24 -void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
   22.25 -			 dma_addr_t dma_handle);
   22.26 -
   22.27 -extern dma_addr_t dma_map_single(struct device *hwdev, void *ptr, size_t size,
   22.28 -				 enum dma_data_direction direction);
   22.29 -extern void dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size,
   22.30 -			     enum dma_data_direction direction);
   22.31 -
   22.32 -#define dma_map_page(dev,page,offset,size,dir) \
   22.33 -	dma_map_single((dev), page_address(page)+(offset), (size), (dir))
   22.34 -
   22.35 -extern void
   22.36 -dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size,
   22.37 -			enum dma_data_direction direction);
   22.38 -
   22.39 -extern void
   22.40 -dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size,
   22.41 -                           enum dma_data_direction direction);
   22.42 -
   22.43 -static inline void dma_sync_sg_for_cpu(struct device *hwdev,
   22.44 -				       struct scatterlist *sg,
   22.45 -				       int nelems, int direction)
   22.46 -{
   22.47 -	if (direction == DMA_NONE)
   22.48 -		out_of_line_bug();
   22.49 -
   22.50 -	if (swiotlb)
   22.51 -		return swiotlb_sync_sg_for_cpu(hwdev,sg,nelems,direction);
   22.52 -
   22.53 -	flush_write_buffers();
   22.54 -}
   22.55 -
   22.56 -static inline void dma_sync_sg_for_device(struct device *hwdev,
   22.57 -					  struct scatterlist *sg,
   22.58 -					  int nelems, int direction)
   22.59 -{
   22.60 -	if (direction == DMA_NONE)
   22.61 -		out_of_line_bug();
   22.62 -
   22.63 -	if (swiotlb)
   22.64 -		return swiotlb_sync_sg_for_device(hwdev,sg,nelems,direction);
   22.65 -
   22.66 -	flush_write_buffers();
   22.67 -}
   22.68 -
   22.69 -extern int dma_map_sg(struct device *hwdev, struct scatterlist *sg,
   22.70 -		      int nents, int direction);
   22.71 -extern void dma_unmap_sg(struct device *hwdev, struct scatterlist *sg,
   22.72 -			 int nents, int direction);
   22.73 -
   22.74 -#define dma_unmap_page dma_unmap_single
   22.75 -
   22.76 -extern int dma_supported(struct device *hwdev, u64 mask);
   22.77 -extern int dma_get_cache_alignment(void);
   22.78 -#define dma_is_consistent(h) 1
   22.79 -
   22.80 -static inline int dma_set_mask(struct device *dev, u64 mask)
   22.81 -{
   22.82 -	if (!dev->dma_mask || !dma_supported(dev, mask))
   22.83 -		return -EIO;
   22.84 -	*dev->dma_mask = mask;
   22.85 -	return 0;
   22.86 -}
   22.87 -
   22.88 -static inline void dma_cache_sync(void *vaddr, size_t size, enum dma_data_direction dir)
   22.89 -{
   22.90 -	flush_write_buffers();
   22.91 -}
   22.92 -#endif
   22.93 +#include <asm-i386/dma-mapping.h>
    23.1 --- a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h	Tue Aug 16 10:12:18 2005 +0000
    23.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/pci.h	Tue Aug 16 11:20:47 2005 +0000
    23.3 @@ -79,7 +79,9 @@ extern int iommu_sac_force;
    23.4  #else
    23.5  /* No IOMMU */
    23.6  
    23.7 -#define PCI_DMA_BUS_IS_PHYS	1
    23.8 +/* On Xen we use SWIOTLB instead of blk-specific bounce buffers. */
    23.9 +#define PCI_DMA_BUS_IS_PHYS	(0)
   23.10 +
   23.11  #define pci_dac_dma_supported(pci_dev, mask)    1
   23.12  
   23.13  #define DECLARE_PCI_UNMAP_ADDR(ADDR_NAME)
    24.1 --- a/linux-2.6-xen-sparse/include/asm-xen/hypervisor.h	Tue Aug 16 10:12:18 2005 +0000
    24.2 +++ b/linux-2.6-xen-sparse/include/asm-xen/hypervisor.h	Tue Aug 16 11:20:47 2005 +0000
    24.3 @@ -134,7 +134,8 @@ void xen_invlpg_mask(cpumask_t *mask, un
    24.4  #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
    24.5  #endif /* linux < 2.6.0 */
    24.6  
    24.7 -void xen_contig_memory(unsigned long vstart, unsigned int order);
    24.8 +void xen_create_contiguous_region(unsigned long vstart, unsigned int order);
    24.9 +void xen_destroy_contiguous_region(unsigned long vstart, unsigned int order);
   24.10  
   24.11  #ifdef CONFIG_XEN_PHYSDEV_ACCESS
   24.12  /* Allocate a contiguous empty region of low memory. Return virtual start. */