ia64/linux-2.6.18-xen.hg

changeset 696:2b5cc22ab406

xen/dom0: Reassign memory resources to device for pci passthrough.

This patch adds the function that reassign page-aligned memory
resources, to dom0 linux. The function is useful when we assign I/O
device to HVM domain using pci passthrough.

When we assign a device to HVM domain using pci passthrough,
the device needs to be assigned page-aligned memory resources. If the
memory resource is not page-aligned, following error occurs.

Error: pci: 0000:00:1d.7: non-page-aligned MMIO BAR found.

On many system, BIOS assigns memory resources to the device and
enables it. So my patch disables the device, and releases resources,
Then it assigns page-aligned memory resource to the device.

To reassign resources, please add boot parameters of dom0 linux as
follows.

reassign_resources reassigndev=00:1d.7,01:00.0

reassign_resources
Enables reassigning resources.

reassigndev= Specifies devices include I/O device and
PCI-PCI
bridge to reassign resources. PCI-PCI bridge
can be specified, if resource windows need to
be expanded.

Signed-off-by: Yuji Shimada <shimada-yxb@necst.nec.co.jp>
author Keir Fraser <keir.fraser@citrix.com>
date Thu Oct 09 11:10:43 2008 +0100 (2008-10-09)
parents 75622c7943a9
children 9010d63470ff a4b25d49459d
files drivers/pci/Makefile drivers/pci/pci.h drivers/pci/quirks.c drivers/pci/reassigndev.c drivers/pci/setup-bus.c drivers/pci/setup-res.c
line diff
     1.1 --- a/drivers/pci/Makefile	Thu Oct 09 10:11:13 2008 +0100
     1.2 +++ b/drivers/pci/Makefile	Thu Oct 09 11:10:43 2008 +0100
     1.3 @@ -3,7 +3,8 @@
     1.4  #
     1.5  
     1.6  obj-y		+= access.o bus.o probe.o remove.o pci.o quirks.o \
     1.7 -			pci-driver.o search.o pci-sysfs.o rom.o setup-res.o
     1.8 +			pci-driver.o search.o pci-sysfs.o rom.o setup-res.o \
     1.9 +			reassigndev.o
    1.10  obj-$(CONFIG_PROC_FS) += proc.o
    1.11  
    1.12  # Build PCI Express stuff if needed
     2.1 --- a/drivers/pci/pci.h	Thu Oct 09 10:11:13 2008 +0100
     2.2 +++ b/drivers/pci/pci.h	Thu Oct 09 11:10:43 2008 +0100
     2.3 @@ -99,3 +99,8 @@ pci_match_one_device(const struct pci_de
     2.4  	return NULL;
     2.5  }
     2.6  
     2.7 +#define ROUND_UP_TO_PAGESIZE(size) ((size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1))
     2.8 +
     2.9 +extern int reassign_resources;
    2.10 +extern int is_reassigndev(struct pci_dev *dev);
    2.11 +extern void pci_update_bridge(struct pci_dev *dev, int resno);
     3.1 --- a/drivers/pci/quirks.c	Thu Oct 09 10:11:13 2008 +0100
     3.2 +++ b/drivers/pci/quirks.c	Thu Oct 09 11:10:43 2008 +0100
     3.3 @@ -33,6 +33,19 @@ static int __init set_pci_mem_align(char
     3.4  }
     3.5  __setup("pci-mem-align", set_pci_mem_align);
     3.6  
     3.7 +
     3.8 +int reassign_resources = 0;
     3.9 +
    3.10 +static int __init set_reassign_resources(char *str)
    3.11 +{
    3.12 +	/* resources reassign on */
    3.13 +	reassign_resources = 1;
    3.14 +	printk(KERN_DEBUG "PCI: resource reassign ON.\n");
    3.15 +
    3.16 +	return 1;
    3.17 +}
    3.18 +__setup("reassign_resources", set_reassign_resources);
    3.19 +
    3.20  /* This quirk function enables us to force all memory resources which are 
    3.21   * assigned to PCI devices, to be page-aligned.
    3.22   */
    3.23 @@ -42,6 +55,42 @@ static void __devinit quirk_align_mem_re
    3.24  	struct resource *r;
    3.25  	resource_size_t old_start;
    3.26  
    3.27 +	if (reassign_resources) {
    3.28 +		if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
    3.29 +		    (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
    3.30 +			/* PCI Host Bridge isn't a target device */
    3.31 +			return;
    3.32 +		}
    3.33 +		if (is_reassigndev(dev)) {
    3.34 +			printk(KERN_INFO 
    3.35 +				"PCI: Disable device and release resources"
    3.36 +				" [%s].\n", pci_name(dev));
    3.37 +			pci_disable_device(dev);
    3.38 +
    3.39 +			for (i=0; i < PCI_NUM_RESOURCES; i++) {
    3.40 +				r = &dev->resource[i];
    3.41 +				if ((r == NULL) || 
    3.42 +				   !(r->flags & IORESOURCE_MEM))
    3.43 +					continue;
    3.44 +
    3.45 +				r->end = r->end - r->start;
    3.46 +				r->start = 0;
    3.47 +
    3.48 +				if (i < PCI_BRIDGE_RESOURCES) {
    3.49 +					pci_update_resource(dev, r, i);
    3.50 +				} else if (i == 8 || i == 9) {
    3.51 +					/* need to update(clear) the Base/Limit
    3.52 +					 * register also, because PCI bridge is
    3.53 +					 * disabled and the resource is 
    3.54 +					 * released.
    3.55 +					 */
    3.56 +					pci_update_bridge(dev, i);
    3.57 +				}
    3.58 +			}
    3.59 +		}
    3.60 +		return;
    3.61 +	}
    3.62 +
    3.63  	if (!pci_mem_align)
    3.64  		return;
    3.65  
     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/drivers/pci/reassigndev.c	Thu Oct 09 11:10:43 2008 +0100
     4.3 @@ -0,0 +1,80 @@
     4.4 +/*
     4.5 + * Copyright (c) 2008, NEC Corporation.
     4.6 + *
     4.7 + * This program is free software; you can redistribute it and/or modify it
     4.8 + * under the terms and conditions of the GNU General Public License,
     4.9 + * version 2, as published by the Free Software Foundation.
    4.10 + *
    4.11 + * This program is distributed in the hope it will be useful, but WITHOUT
    4.12 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    4.13 + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
    4.14 + * more details.
    4.15 + *
    4.16 + * You should have received a copy of the GNU General Public License along with
    4.17 + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
    4.18 + * Place - Suite 330, Boston, MA 02111-1307 USA.
    4.19 + */
    4.20 +
    4.21 +#include <linux/kernel.h>
    4.22 +#include <linux/pci.h>
    4.23 +#include <linux/string.h>
    4.24 +#include "pci.h"
    4.25 +
    4.26 +
    4.27 +#define	REASSIGNDEV_PARAM_MAX	(2048)
    4.28 +#define	TOKEN_MAX	(12)	/* "SSSS:BB:DD.F" length is 12 */
    4.29 +
    4.30 +static char param_reassigndev[REASSIGNDEV_PARAM_MAX] = {0};
    4.31 +
    4.32 +static int __init reassigndev_setup(char *str)
    4.33 +{
    4.34 +	strncpy(param_reassigndev, str, REASSIGNDEV_PARAM_MAX);
    4.35 +	param_reassigndev[REASSIGNDEV_PARAM_MAX - 1] = '\0';
    4.36 +	return 1;
    4.37 +}
    4.38 +__setup("reassigndev=", reassigndev_setup);
    4.39 +
    4.40 +int is_reassigndev(struct pci_dev *dev)
    4.41 +{
    4.42 +	char dev_str[TOKEN_MAX+1];
    4.43 +	int seg, bus, slot, func;
    4.44 +	int len;
    4.45 +	char *p, *next_str;
    4.46 +
    4.47 +	p = param_reassigndev;
    4.48 +	for (; p; p = next_str + 1) {
    4.49 +		next_str = strpbrk(p, ",");
    4.50 +		if (next_str) {
    4.51 +			len = next_str - p;
    4.52 +		} else {
    4.53 +			len = strlen(p);
    4.54 +		}
    4.55 +		if (len > 0 && len <= TOKEN_MAX) {
    4.56 +			strncpy(dev_str, p, len);
    4.57 +			*(dev_str + len) = '\0';
    4.58 +
    4.59 +			if (sscanf(dev_str, "%x:%x:%x.%x", 
    4.60 +				&seg, &bus, &slot, &func) != 4) {
    4.61 +				if (sscanf(dev_str, "%x:%x.%x", 
    4.62 +					&bus, &slot, &func) == 3) {
    4.63 +					seg = 0;
    4.64 +				} else {
    4.65 +					/* failed to scan strings */
    4.66 +					seg = -1;
    4.67 +					bus = -1;
    4.68 +				}
    4.69 +			}
    4.70 +			if (seg == pci_domain_nr(dev->bus) &&
    4.71 +			    bus == dev->bus->number &&
    4.72 +			    slot == PCI_SLOT(dev->devfn) &&
    4.73 +			    func == PCI_FUNC(dev->devfn)) {
    4.74 +				/* It's a target device */
    4.75 +				return 1;
    4.76 +			}
    4.77 +		}
    4.78 +		if (!next_str)
    4.79 +			break;
    4.80 +	}
    4.81 +
    4.82 +	return 0;
    4.83 +}
     5.1 --- a/drivers/pci/setup-bus.c	Thu Oct 09 10:11:13 2008 +0100
     5.2 +++ b/drivers/pci/setup-bus.c	Thu Oct 09 11:10:43 2008 +0100
     5.3 @@ -26,6 +26,7 @@
     5.4  #include <linux/cache.h>
     5.5  #include <linux/slab.h>
     5.6  
     5.7 +#include "pci.h"
     5.8  
     5.9  #define DEBUG_CONFIG 1
    5.10  #if DEBUG_CONFIG
    5.11 @@ -344,7 +345,8 @@ pbus_size_mem(struct pci_bus *bus, unsig
    5.12  
    5.13  	list_for_each_entry(dev, &bus->devices, bus_list) {
    5.14  		int i;
    5.15 -		
    5.16 +		int reassign = reassign_resources ? is_reassigndev(dev) : 0;
    5.17 +
    5.18  		for (i = 0; i < PCI_NUM_RESOURCES; i++) {
    5.19  			struct resource *r = &dev->resource[i];
    5.20  			unsigned long r_size;
    5.21 @@ -352,6 +354,11 @@ pbus_size_mem(struct pci_bus *bus, unsig
    5.22  			if (r->parent || (r->flags & mask) != type)
    5.23  				continue;
    5.24  			r_size = r->end - r->start + 1;
    5.25 +
    5.26 +			if (reassign) {
    5.27 +				r_size = ROUND_UP_TO_PAGESIZE(r_size);
    5.28 +			}
    5.29 +
    5.30  			/* For bridges size != alignment */
    5.31  			align = (i < PCI_BRIDGE_RESOURCES) ? r_size : r->start;
    5.32  			order = __ffs(align) - 20;
     6.1 --- a/drivers/pci/setup-res.c	Thu Oct 09 10:11:13 2008 +0100
     6.2 +++ b/drivers/pci/setup-res.c	Thu Oct 09 11:10:43 2008 +0100
     6.3 @@ -117,19 +117,96 @@ pci_claim_resource(struct pci_dev *dev, 
     6.4  }
     6.5  EXPORT_SYMBOL_GPL(pci_claim_resource);
     6.6  
     6.7 +void 
     6.8 +pci_update_bridge(struct pci_dev *dev, int resno)
     6.9 +{
    6.10 +	struct resource *res = &dev->resource[resno]; 
    6.11 +	struct pci_bus_region region;
    6.12 +	u32 l, dw, base_up32, limit_up32;
    6.13 +
    6.14 +	if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE ||
    6.15 +	    (dev->class >> 8) != PCI_CLASS_BRIDGE_PCI) {
    6.16 +		return;
    6.17 +	}
    6.18 +
    6.19 +	if (!res->flags)
    6.20 +		return;
    6.21 +
    6.22 +	switch (resno) {
    6.23 +	case 8 :	/* MMIO Base/Limit */
    6.24 +		pcibios_resource_to_bus(dev, &region, res);
    6.25 +		if (res->flags & IORESOURCE_MEM &&
    6.26 +		    !(res->flags & IORESOURCE_PREFETCH)) {
    6.27 +			l = (region.start >> 16) & 0xfff0;
    6.28 +			l |= region.end & 0xfff00000;
    6.29 +		} else {
    6.30 +			l = 0x0000fff0;
    6.31 +		}
    6.32 +		pci_write_config_dword(dev, PCI_MEMORY_BASE, l);
    6.33 +
    6.34 +		break;
    6.35 +
    6.36 +	case 9 :	/* Prefetchable MMIO Base/Limit */
    6.37 +		/* Clear out the upper 32 bits of PREF limit.
    6.38 +		 * If PCI_PREF_BASE_UPPER32 was non-zero, this temporarily
    6.39 +		 * disables PREF range, which is ok.
    6.40 +		 */
    6.41 +		pci_write_config_dword(dev, PCI_PREF_LIMIT_UPPER32, 0);
    6.42 +
    6.43 +		/* Get PREF 32/64 bits Addressing mode */
    6.44 +		pci_read_config_dword(dev, PCI_PREF_MEMORY_BASE, &dw);
    6.45 +
    6.46 +		pcibios_resource_to_bus(dev, &region, res);
    6.47 +		if (res->flags & IORESOURCE_MEM &&
    6.48 +		    res->flags & IORESOURCE_PREFETCH) {
    6.49 +			l = (region.start >> 16) & 0xfff0;
    6.50 +			l |= region.end & 0xfff00000;
    6.51 +
    6.52 +			if (dw & PCI_PREF_RANGE_TYPE_64) {
    6.53 +				base_up32 = (region.start >> 32) & 0xffffffff;
    6.54 +				limit_up32 = (region.end >> 32) & 0xffffffff;
    6.55 +			} else {
    6.56 +				base_up32 = 0;
    6.57 +				limit_up32 = 0;
    6.58 +			}
    6.59 +		} else {
    6.60 +			l = 0x0000fff0;
    6.61 +			base_up32 = 0xffffffff;
    6.62 +			limit_up32 = 0;
    6.63 +		}
    6.64 +		pci_write_config_dword(dev, PCI_PREF_MEMORY_BASE, l);
    6.65 +		/* Set up the upper 32 bits of PREF base/limit. */
    6.66 +		pci_write_config_dword(dev, PCI_PREF_BASE_UPPER32, base_up32);
    6.67 +		pci_write_config_dword(dev, PCI_PREF_LIMIT_UPPER32, limit_up32);
    6.68 +		break;
    6.69 +	default :
    6.70 +		BUG();
    6.71 +		break;
    6.72 +	}
    6.73 +}
    6.74 +
    6.75  int pci_assign_resource(struct pci_dev *dev, int resno)
    6.76  {
    6.77  	struct pci_bus *bus = dev->bus;
    6.78  	struct resource *res = dev->resource + resno;
    6.79  	resource_size_t size, min, align;
    6.80  	int ret;
    6.81 +	int reassigndev = reassign_resources ? is_reassigndev(dev) : 0;
    6.82  
    6.83  	size = res->end - res->start + 1;
    6.84  	min = (res->flags & IORESOURCE_IO) ? PCIBIOS_MIN_IO : PCIBIOS_MIN_MEM;
    6.85  	/* The bridge resources are special, as their
    6.86  	   size != alignment. Sizing routines return
    6.87  	   required alignment in the "start" field. */
    6.88 -	align = (resno < PCI_BRIDGE_RESOURCES) ? size : res->start;
    6.89 +	if (resno < PCI_BRIDGE_RESOURCES) {
    6.90 +		align = size;
    6.91 +		if ((reassigndev) &&
    6.92 +		    (res->flags & IORESOURCE_MEM)) {
    6.93 +			align = ROUND_UP_TO_PAGESIZE(align);
    6.94 +		}
    6.95 +	} else {
    6.96 +		align = res->start;
    6.97 +	}
    6.98  
    6.99  	/* First, try exact prefetching match.. */
   6.100  	ret = pci_bus_alloc_resource(bus, res, size, align, min,
   6.101 @@ -154,6 +231,9 @@ int pci_assign_resource(struct pci_dev *
   6.102  			resno, (unsigned long long)size,
   6.103  			(unsigned long long)res->start, pci_name(dev));
   6.104  	} else if (resno < PCI_BRIDGE_RESOURCES) {
   6.105 +		printk(KERN_DEBUG "PCI: Assign resource(%d) on %s "
   6.106 +			"%016llx - %016llx\n", resno, pci_name(dev),
   6.107 +			(u64)res->start, (u64)res->end);
   6.108  		pci_update_resource(dev, res, resno);
   6.109  	}
   6.110