ia64/xen-unstable

changeset 18049:e61978c24d84

xend: vt-d: improved FLR logic for pass-thru PCI devices

1) If the device is PCIe endpoint and supports PCIe FLR, we use that;
else

2) if the device is PCIe endpoint, and all functions on the
device are assigned to the same guest, we use the immediate parent
bus's Secondary Bus Reset to reset all functions of the device (here,
actually we require all the functions of the device be assigned to the
same guest); else

3) if the device is PCI endpoint and is on a host bus
(e.g. integrated devices), and if the device supports PCI Advanced
Capabilities, we use that for FLR; else

4) we use the Secondary Bus Reset (if the PCI device is behind a
PCI/PCI-X bridge, then all devices behind the uppermost such PCI/PCI-X
bridge above this device must be co-assigned).

Signed-off-by: Dexuan Cui <dexuan.cui@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Jul 14 10:11:39 2008 +0100 (2008-07-14)
parents caba894b265f
children 39c2cab9e765
files tools/python/xen/util/pci.py tools/python/xen/xend/XendDomainInfo.py tools/python/xen/xend/server/DevController.py tools/python/xen/xend/server/pciif.py
line diff
     1.1 --- a/tools/python/xen/util/pci.py	Mon Jul 14 10:10:14 2008 +0100
     1.2 +++ b/tools/python/xen/util/pci.py	Mon Jul 14 10:11:39 2008 +0100
     1.3 @@ -10,6 +10,8 @@ import os, os.path
     1.4  import resource
     1.5  import re
     1.6  import types
     1.7 +import struct
     1.8 +import time
     1.9  
    1.10  PROC_MNT_PATH = '/proc/mounts'
    1.11  PROC_PCI_PATH = '/proc/bus/pci/devices'
    1.12 @@ -28,8 +30,44 @@ SYSFS_PCI_DEV_CLASS_PATH = '/class'
    1.13  
    1.14  LSPCI_CMD = 'lspci'
    1.15  
    1.16 +PCI_DEV_REG_EXPRESS_STR = r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}."+ \
    1.17 +            r"[0-9a-fA-F]{1}"
    1.18 +PCI_DEV_FORMAT_STR = '%04x:%02x:%02x.%01x'
    1.19 +
    1.20 +DEV_TYPE_PCIe_ENDPOINT  = 0
    1.21 +DEV_TYPE_PCIe_BRIDGE    = 1
    1.22 +DEV_TYPE_PCI_BRIDGE     = 2
    1.23 +DEV_TYPE_PCI            = 3    
    1.24 +
    1.25 +PCI_STATUS = 0x6
    1.26 +PCI_CLASS_DEVICE = 0x0a
    1.27 +PCI_CLASS_BRIDGE_PCI = 0x0604
    1.28 +
    1.29 +PCI_CAPABILITY_LIST = 0x34
    1.30 +PCI_CB_BRIDGE_CONTROL = 0x3e
    1.31 +PCI_BRIDGE_CTL_BUS_RESET= 0x40
    1.32 +
    1.33 +PCI_CAP_ID_EXP = 0x10
    1.34 +PCI_EXP_FLAGS  = 0x2
    1.35 +PCI_EXP_FLAGS_TYPE = 0x00f0
    1.36 +PCI_EXP_TYPE_PCI_BRIDGE = 0x7
    1.37 +PCI_EXP_DEVCAP = 0x4
    1.38 +PCI_EXP_DEVCAP_FLR = (0x1 << 28)
    1.39 +PCI_EXP_DEVCTL = 0x8
    1.40 +PCI_EXP_DEVCTL_FLR = (0x1 << 15)
    1.41 +
    1.42 +PCI_CAP_ID_AF = 0x13
    1.43 +PCI_AF_CAPs   = 0x3
    1.44 +PCI_AF_CAPs_TP_FLR = 0x3
    1.45 +PCI_AF_CTL = 0x4
    1.46 +PCI_AF_CTL_FLR = 0x1
    1.47 +
    1.48 +PCI_BAR_0 = 0x10
    1.49 +PCI_BAR_5 = 0x24
    1.50 +PCI_BAR_SPACE = 0x01
    1.51  PCI_BAR_IO = 0x01
    1.52  PCI_BAR_IO_MASK = ~0x03
    1.53 +PCI_BAR_MEM = 0x00
    1.54  PCI_BAR_MEM_MASK = ~0x0f
    1.55  PCI_STATUS_CAP_MASK = 0x10
    1.56  PCI_STATUS_OFFSET = 0x6
    1.57 @@ -65,6 +103,17 @@ def parse_hex(val):
    1.58      except ValueError:
    1.59          return None
    1.60  
    1.61 +def parse_pci_name(pci_name_string):
    1.62 +    # Format: xxxx:xx:xx:x
    1.63 +    s = pci_name_string
    1.64 +    s = s.split(':')
    1.65 +    dom = parse_hex(s[0])
    1.66 +    bus = parse_hex(s[1])
    1.67 +    s = s[2].split('.')
    1.68 +    dev = parse_hex(s[0])
    1.69 +    func =  parse_hex(s[1])
    1.70 +    return (dom, bus, dev, func)
    1.71 +
    1.72  def find_sysfs_mnt():
    1.73      global sysfs_mnt_point
    1.74      if not sysfs_mnt_point is None:
    1.75 @@ -134,13 +183,40 @@ def create_lspci_info():
    1.76          if device_name is not None:
    1.77              lspci_info[device_name] = device_info
    1.78  
    1.79 +def save_pci_conf_space(devs_string):
    1.80 +    pci_list = []
    1.81 +    cfg_list = []
    1.82 +    sysfs_mnt = find_sysfs_mnt()
    1.83 +    for pci_str in devs_string:
    1.84 +        pci_path = sysfs_mnt + SYSFS_PCI_DEVS_PATH + '/' + pci_str + \
    1.85 +                SYSFS_PCI_DEV_CONFIG_PATH
    1.86 +        fd = os.open(pci_path, os.O_RDONLY)
    1.87 +        configs = []
    1.88 +        for i in range(0, 256, 4):
    1.89 +            configs = configs + [os.read(fd,4)]
    1.90 +        os.close(fd)
    1.91 +        pci_list = pci_list + [pci_path]
    1.92 +        cfg_list = cfg_list + [configs]
    1.93 +    return (pci_list, cfg_list)
    1.94 +
    1.95 +def restore_pci_conf_space(pci_cfg_list):
    1.96 +    pci_list = pci_cfg_list[0]
    1.97 +    cfg_list = pci_cfg_list[1]
    1.98 +    for i in range(0, len(pci_list)):
    1.99 +        pci_path = pci_list[i]
   1.100 +        configs  = cfg_list[i]
   1.101 +        fd = os.open(pci_path, os.O_WRONLY)
   1.102 +        for dw in configs:
   1.103 +            os.write(fd, dw)
   1.104 +        os.close(fd) 
   1.105 +
   1.106  class PciDeviceNotFoundError(Exception):
   1.107      def __init__(self,domain,bus,slot,func):
   1.108          self.domain = domain
   1.109          self.bus = bus
   1.110          self.slot = slot
   1.111          self.func = func
   1.112 -        self.name = "%04x:%02x:%02x.%01x"%(domain, bus, slot, func)
   1.113 +        self.name = PCI_DEV_FORMAT_STR %(domain, bus, slot, func)
   1.114      
   1.115      def __str__(self):
   1.116          return ('PCI Device %s Not Found' % (self.name))
   1.117 @@ -151,13 +227,29 @@ class PciDeviceParseError(Exception):
   1.118      def __str__(self):
   1.119          return 'Error Parsing PCI Device Info: '+self.message
   1.120  
   1.121 +class PciDeviceAssignmentError(Exception):
   1.122 +    def __init__(self,msg):
   1.123 +        self.message = msg
   1.124 +    def __str__(self):
   1.125 +        return 'pci: impproper device assignment spcified: ' + \
   1.126 +            self.message
   1.127 +
   1.128 +class PciDeviceFlrError(PciDeviceAssignmentError):
   1.129 +    def __init__(self,msg):
   1.130 +        self.message = msg
   1.131 +    def __str__(self):
   1.132 +        return 'Can not find a suitable FLR method for the device(s): ' + \
   1.133 +            self.message
   1.134 +
   1.135  class PciDevice:
   1.136      def __init__(self, domain, bus, slot, func):
   1.137          self.domain = domain
   1.138          self.bus = bus
   1.139          self.slot = slot
   1.140          self.func = func
   1.141 -        self.name = "%04x:%02x:%02x.%01x"%(domain, bus, slot, func)
   1.142 +        self.name = PCI_DEV_FORMAT_STR % (domain, bus, slot, func)
   1.143 +        self.cfg_space_path = find_sysfs_mnt()+SYSFS_PCI_DEVS_PATH+'/'+ \
   1.144 +            self.name + SYSFS_PCI_DEV_CONFIG_PATH 
   1.145          self.irq = 0
   1.146          self.iomem = []
   1.147          self.ioports = []
   1.148 @@ -175,9 +267,317 @@ class PciDevice:
   1.149          self.classname = ""
   1.150          self.subvendorname = ""
   1.151          self.subdevicename = ""
   1.152 +        self.dev_type = None
   1.153 +        self.has_non_page_aligned_bar = False
   1.154 +        self.pcie_flr = False
   1.155 +        self.pci_af_flr = False
   1.156 +        self.detect_dev_info()
   1.157          self.get_info_from_sysfs()
   1.158          self.get_info_from_lspci()
   1.159  
   1.160 +    def find_parent(self):
   1.161 +        # i.e.,  /sys/bus/pci/devices/0000:00:19.0 or
   1.162 +        #        /sys/bus/pci/devices/0000:03:04.0
   1.163 +        path = find_sysfs_mnt()+SYSFS_PCI_DEVS_PATH+'/'+ self.name
   1.164 +        # i.e., ../../../devices/pci0000:00/0000:00:19.0
   1.165 +        #  ../../../devices/pci0000:00/0000:00:02.0/0000:01:00.2/0000:03:04.0
   1.166 +        try:
   1.167 +            target = os.readlink(path)
   1.168 +            lst = target.split('/')
   1.169 +            parent = lst[len(lst)-2]
   1.170 +            if parent[0:3] == 'pci':
   1.171 +                parent = parent[3:]
   1.172 +                lst = parent.split(':')
   1.173 +                dom = int(lst[0], 16)
   1.174 +                bus = int(lst[1], 16)
   1.175 +                dev = 0
   1.176 +                func = 0
   1.177 +            else:
   1.178 +                lst = parent.split(':')
   1.179 +                dom = int(lst[0], 16)
   1.180 +                bus = int(lst[1], 16)
   1.181 +                lst = lst[2]
   1.182 +                lst = lst.split('.')
   1.183 +                dev =  int(lst[0], 16)
   1.184 +                func =  int(lst[1], 16)
   1.185 +            return (dom, bus, dev, func)
   1.186 +        except OSError, (errno, strerr):
   1.187 +            raise PciDeviceParseError('Can not locate the parent of %s',
   1.188 +                self.name)
   1.189 +
   1.190 +    def find_the_uppermost_pci_bridge(self):
   1.191 +        # Find the uppermost PCI/PCI-X bridge
   1.192 +        (dom, b, d, f) = self.find_parent()
   1.193 +        dev = dev_parent = PciDevice(dom, b, d, f)
   1.194 +        while dev_parent.dev_type != DEV_TYPE_PCIe_BRIDGE:
   1.195 +            (dom, b, d, f) = dev_parent.find_parent()
   1.196 +            dev = dev_parent
   1.197 +            dev_parent = PciDevice(dom, b, d, f)
   1.198 +        return dev
   1.199 +
   1.200 +    def find_all_devices_behind_the_bridge(self, ignore_bridge):
   1.201 +        sysfs_mnt = find_sysfs_mnt()
   1.202 +        self_path = sysfs_mnt + SYSFS_PCI_DEVS_PATH + '/' + self.name
   1.203 +        pci_names = os.popen('ls ' + self_path).read()
   1.204 +        dev_list = re.findall(PCI_DEV_REG_EXPRESS_STR, pci_names)
   1.205 +
   1.206 +        list = [self.name]
   1.207 +        for pci_str in dev_list:
   1.208 +            (dom, b, d, f) = parse_pci_name(pci_str)
   1.209 +            dev = PciDevice(dom, b, d, f)
   1.210 +            if dev.dev_type == DEV_TYPE_PCI_BRIDGE or \
   1.211 +                dev.dev_type == DEV_TYPE_PCIe_BRIDGE:
   1.212 +                sub_list_including_self = \
   1.213 +                    dev.find_all_devices_behind_the_bridge(ignore_bridge)
   1.214 +                if ignore_bridge:
   1.215 +                    del sub_list_including_self[0]
   1.216 +                list = list + [sub_list_including_self]
   1.217 +            else:
   1.218 +                list = list + [dev.name]
   1.219 +        return list
   1.220 +        
   1.221 +    def find_coassigned_devices(self, ignore_bridge = True):
   1.222 +        ''' Here'self' is a PCI device, we need find the uppermost PCI/PCI-X
   1.223 +            bridge, and all devices behind it must be co-assigned to the same
   1.224 +            guest.
   1.225 +        
   1.226 +            Parameter:
   1.227 +                [ignore_bridge]: if set, the returned result doesn't include
   1.228 +            any bridge behind the uppermost PCI/PCI-X bridge.
   1.229 +        
   1.230 +            Note: The first element of the return value is the uppermost
   1.231 +                PCI/PCI-X bridge. If the caller doesn't need the first
   1.232 +                element,  the caller itself can remove it explicitly.
   1.233 +        '''
   1.234 +        dev = self.find_the_uppermost_pci_bridge()
   1.235 +        dev_list = dev.find_all_devices_behind_the_bridge(ignore_bridge)
   1.236 +        dev_list = re.findall(PCI_DEV_REG_EXPRESS_STR, '%s' % dev_list)
   1.237 +        return dev_list
   1.238 +
   1.239 +    def do_secondary_bus_reset(self, target_bus, devs):
   1.240 +        # Save the config spaces of all the devices behind the bus.
   1.241 +        (pci_list, cfg_list) = save_pci_conf_space(devs)
   1.242 +        
   1.243 +        #Do the Secondary Bus Reset
   1.244 +        sysfs_mnt = find_sysfs_mnt()
   1.245 +        parent_path = sysfs_mnt + SYSFS_PCI_DEVS_PATH + '/' + \
   1.246 +            target_bus + SYSFS_PCI_DEV_CONFIG_PATH
   1.247 +        fd = os.open(parent_path, os.O_WRONLY)
   1.248 +        # Assert Secondary Bus Reset
   1.249 +        os.lseek(fd, PCI_CB_BRIDGE_CONTROL, 0)
   1.250 +        os.write(fd, struct.pack('I', PCI_BRIDGE_CTL_BUS_RESET))
   1.251 +        time.sleep(0.200)
   1.252 +        # De-assert Secondary Bus Reset
   1.253 +        os.lseek(fd, 0x3e, 0)
   1.254 +        os.write(fd, struct.pack('I', 0x00))
   1.255 +        time.sleep(0.200)
   1.256 +        os.close(fd)
   1.257 +
   1.258 +        # Restore the config spaces
   1.259 +        restore_pci_conf_space((pci_list, cfg_list))
   1.260 +        
   1.261 +    def find_all_the_multi_functions(self):
   1.262 +        sysfs_mnt = find_sysfs_mnt()
   1.263 +        pci_names = os.popen('ls ' + sysfs_mnt + SYSFS_PCI_DEVS_PATH).read()
   1.264 +        p = self.name
   1.265 +        p = p[0 : p.rfind('.')] + '.[0-7]'
   1.266 +        funcs = re.findall(p, pci_names)
   1.267 +        return funcs
   1.268 +
   1.269 +    def find_cap_offset(self, cap):
   1.270 +        path = find_sysfs_mnt()+SYSFS_PCI_DEVS_PATH+'/'+ \
   1.271 +               self.name+SYSFS_PCI_DEV_CONFIG_PATH
   1.272 +
   1.273 +        pos = PCI_CAPABILITY_LIST
   1.274 +
   1.275 +        try:
   1.276 +            fd = os.open(path, os.O_RDONLY)
   1.277 +            os.lseek(fd, PCI_STATUS, 0)
   1.278 +            status = struct.unpack('H', os.read(fd, 2))[0]
   1.279 +            if (status & 0x10) == 0:
   1.280 +                # The device doesn't support PCI_STATUS_CAP_LIST
   1.281 +                return 0
   1.282 +
   1.283 +            max_cap = 48
   1.284 +            while max_cap > 0:
   1.285 +                os.lseek(fd, pos, 0)
   1.286 +                pos = ord(os.read(fd, 1))
   1.287 +                if pos < 0x40:
   1.288 +                    pos = 0
   1.289 +                    break;
   1.290 +                os.lseek(fd, pos + 0, 0)
   1.291 +                id = ord(os.read(fd, 1))
   1.292 +                if id == 0xff:
   1.293 +                    pos = 0
   1.294 +                    break;
   1.295 +
   1.296 +                # Found the capability
   1.297 +                if id == cap:
   1.298 +                    break;
   1.299 +
   1.300 +                # Test the next one
   1.301 +                pos = pos + 1
   1.302 +                max_cap = max_cap - 1;
   1.303 +
   1.304 +            os.close(fd)
   1.305 +        except OSError, (errno, strerr):
   1.306 +            raise PciDeviceParseError(('Error when accessing sysfs: %s (%d)' %
   1.307 +                (strerr, errno)))
   1.308 +        return pos
   1.309 +
   1.310 +    def pci_conf_read8(self, pos):
   1.311 +        fd = os.open(self.cfg_space_path, os.O_RDONLY)
   1.312 +        os.lseek(fd, pos, 0)
   1.313 +        str = os.read(fd, 1)
   1.314 +        os.close(fd)
   1.315 +        val = struct.unpack('B', str)[0]
   1.316 +        return val
   1.317 +
   1.318 +    def pci_conf_read16(self, pos):
   1.319 +        fd = os.open(self.cfg_space_path, os.O_RDONLY)
   1.320 +        os.lseek(fd, pos, 0)
   1.321 +        str = os.read(fd, 2)
   1.322 +        os.close(fd)
   1.323 +        val = struct.unpack('H', str)[0]
   1.324 +        return val
   1.325 +
   1.326 +    def pci_conf_read32(self, pos):
   1.327 +        fd = os.open(self.cfg_space_path, os.O_RDONLY)
   1.328 +        os.lseek(fd, pos, 0)
   1.329 +        str = os.read(fd, 4)
   1.330 +        os.close(fd)
   1.331 +        val = struct.unpack('I', str)[0]
   1.332 +        return val
   1.333 +
   1.334 +    def pci_conf_write8(self, pos, val):
   1.335 +        str = struct.pack('B', val)
   1.336 +        fd = os.open(self.cfg_space_path, os.O_WRONLY)
   1.337 +        os.lseek(fd, pos, 0)
   1.338 +        os.write(fd, str)
   1.339 +        os.close(fd)
   1.340 +
   1.341 +    def pci_conf_write16(self, pos, val):
   1.342 +        str = struct.pack('H', val)
   1.343 +        fd = os.open(self.cfg_space_path, os.O_WRONLY)
   1.344 +        os.lseek(fd, pos, 0)
   1.345 +        os.write(fd, str)
   1.346 +        os.close(fd)
   1.347 +
   1.348 +    def pci_conf_write32(self, pos, val):
   1.349 +        str = struct.pack('I', val)
   1.350 +        fd = os.open(self.cfg_space_path, os.O_WRONLY)
   1.351 +        os.lseek(fd, pos, 0)
   1.352 +        os.write(fd, str)
   1.353 +        os.close(fd)
   1.354 +
   1.355 +    def detect_dev_info(self):
   1.356 +        class_dev = self.pci_conf_read16(PCI_CLASS_DEVICE)
   1.357 +        pos = self.find_cap_offset(PCI_CAP_ID_EXP)
   1.358 +        if class_dev == PCI_CLASS_BRIDGE_PCI:
   1.359 +            if pos == 0:
   1.360 +                self.dev_type = DEV_TYPE_PCI_BRIDGE
   1.361 +            else:
   1.362 +                creg = self.pci_conf_read16(pos + PCI_EXP_FLAGS)
   1.363 +                if ((creg & PCI_EXP_TYPE_PCI_BRIDGE) >> 4) == \
   1.364 +                    PCI_EXP_TYPE_PCI_BRIDGE:
   1.365 +                    self.dev_type = DEV_TYPE_PCI_BRIDGE
   1.366 +                else:
   1.367 +                    self.dev_type = DEV_TYPE_PCIe_BRIDGE
   1.368 +        else:
   1.369 +            if  pos != 0:
   1.370 +                self.dev_type = DEV_TYPE_PCIe_ENDPOINT
   1.371 +            else:
   1.372 +                self.dev_type = DEV_TYPE_PCI
   1.373 +                
   1.374 +        # Force 0000:00:00.0 to be DEV_TYPE_PCIe_BRIDGE
   1.375 +        if self.name == '0000:00:00.0':
   1.376 +            self.dev_type = DEV_TYPE_PCIe_BRIDGE
   1.377 +
   1.378 +        if (self.dev_type == DEV_TYPE_PCI_BRIDGE) or \
   1.379 +            (self.dev_type == DEV_TYPE_PCIe_BRIDGE):
   1.380 +            return
   1.381 +
   1.382 +        # Try to findthe PCIe FLR capability
   1.383 +        if self.dev_type == DEV_TYPE_PCIe_ENDPOINT:
   1.384 +            dev_cap = self.pci_conf_read32(pos + PCI_EXP_DEVCAP)
   1.385 +            if dev_cap & PCI_EXP_DEVCAP_FLR:
   1.386 +                self.pcie_flr = True
   1.387 +        elif self.dev_type == DEV_TYPE_PCI:
   1.388 +            # Try to find the "PCI Advanced Capabilities"
   1.389 +            pos = self.find_cap_offset(PCI_CAP_ID_AF)
   1.390 +            if pos != 0:
   1.391 +                af_cap = self.pci_conf_read8(pos + PCI_AF_CAPs)
   1.392 +                if (af_cap & PCI_AF_CAPs_TP_FLR) == PCI_AF_CAPs_TP_FLR:
   1.393 +                    self.pci_af_flr = True
   1.394 +
   1.395 +        bar_addr = PCI_BAR_0
   1.396 +        while bar_addr <= PCI_BAR_5:
   1.397 +            bar = self.pci_conf_read32(bar_addr)
   1.398 +            if (bar & PCI_BAR_SPACE) == PCI_BAR_MEM:
   1.399 +                bar = bar & PCI_BAR_MEM_MASK
   1.400 +                bar = bar & ~PAGE_MASK
   1.401 +                if bar != 0:
   1.402 +                    self.has_non_page_aligned_bar = True
   1.403 +                    break 
   1.404 +            bar_addr = bar_addr + 4
   1.405 +
   1.406 +    def devs_check_driver(self, devs):
   1.407 +        if len(devs) == 0:
   1.408 +            return
   1.409 +        for pci_dev in devs:
   1.410 +            (dom, b, d, f) = parse_pci_name(pci_dev)
   1.411 +            dev = PciDevice(dom, b, d, f)
   1.412 +            if dev.driver == 'pciback':
   1.413 +                continue
   1.414 +            err_msg = 'pci: %s must be co-assigned to the same guest with %s' + \
   1.415 +                ', but it is not owned by pciback.'
   1.416 +            raise PciDeviceAssignmentError(err_msg % (pci_dev, self.name))
   1.417 +
   1.418 +    def do_FLR(self):
   1.419 +        """ Perform FLR (Functional Level Reset) for the device.
   1.420 +        """
   1.421 +        if self.dev_type == DEV_TYPE_PCIe_ENDPOINT:
   1.422 +            # If PCIe device supports FLR, we use it.
   1.423 +            if self.pcie_flr:
   1.424 +                (pci_list, cfg_list) = save_pci_conf_space([self.name])
   1.425 +                pos = self.find_cap_offset(PCI_CAP_ID_EXP)
   1.426 +                self.pci_conf_write32(pos + PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_FLR)
   1.427 +                # We must sleep at least 100ms for the completion of FLR
   1.428 +                time.sleep(0.200)
   1.429 +                restore_pci_conf_space((pci_list, cfg_list))
   1.430 +            else:
   1.431 +                funcs = self.find_all_the_multi_functions()
   1.432 +                self.devs_check_driver(funcs)
   1.433 +
   1.434 +                parent = '%04x:%02x:%02x.%01x' % self.find_parent()
   1.435 +
   1.436 +                # Do Secondary Bus Reset.
   1.437 +                self.do_secondary_bus_reset(parent, funcs)
   1.438 +        # PCI devices
   1.439 +        else:
   1.440 +            # For PCI device on host bus, we test "PCI Advanced Capabilities".
   1.441 +            if self.bus == 0 and self.pci_af_flr:
   1.442 +                (pci_list, cfg_list) = save_pci_conf_space([self.name])
   1.443 +                # We use Advanced Capability to do FLR.
   1.444 +                pos = self.find_cap_offset(PCI_CAP_ID_AF)
   1.445 +                self.pci_conf_write8(pos + PCI_AF_CTL, PCI_AF_CTL_FLR)
   1.446 +                time.sleep(0.200)
   1.447 +                restore_pci_conf_space((pci_list, cfg_list))
   1.448 +            else:
   1.449 +                if self.bus == 0:
   1.450 +                    err_msg = 'pci: %s is not assignable: it is on bus 0, '+ \
   1.451 +                        'but it has no PCI Advanced Capabilities.'
   1.452 +                    raise PciDeviceFlrError(err_msg % self.name)
   1.453 +                else:
   1.454 +                    devs = self.find_coassigned_devices(False)
   1.455 +                    # Remove the element 0 which is a bridge
   1.456 +                    target_bus = devs[0]
   1.457 +                    del devs[0]
   1.458 +                    self.devs_check_driver(devs)
   1.459 +
   1.460 +                    # Do Secondary Bus Reset.
   1.461 +                    self.do_secondary_bus_reset(target_bus, devs)
   1.462 +
   1.463      def find_capability(self, type):
   1.464          sysfs_mnt = find_sysfs_mnt()
   1.465          if sysfs_mnt == None:
   1.466 @@ -364,7 +764,7 @@ class PciDevice:
   1.467  
   1.468  def main():
   1.469      if len(sys.argv)<5:
   1.470 -        print "Usage: %s <domain> <bus> <slot> <func>\n"
   1.471 +        print "Usage: %s <domain> <bus> <slot> <func>\n" % sys.argv[0]
   1.472          sys.exit(2)
   1.473  
   1.474      dev = PciDevice(int(sys.argv[1],16), int(sys.argv[2],16),
     2.1 --- a/tools/python/xen/xend/XendDomainInfo.py	Mon Jul 14 10:10:14 2008 +0100
     2.2 +++ b/tools/python/xen/xend/XendDomainInfo.py	Mon Jul 14 10:11:39 2008 +0100
     2.3 @@ -701,7 +701,7 @@ class XendDomainInfo:
     2.4                          vslt = x['vslt']
     2.5                          break
     2.6                  if vslt == '0x0':
     2.7 -                    raise VmError("Device %04x:%02x:%02x.%02x is not connected"
     2.8 +                    raise VmError("Device %04x:%02x:%02x.%01x is not connected"
     2.9                                    % (int(dev['domain'],16), int(dev['bus'],16),
    2.10                                       int(dev['slot'],16), int(dev['func'],16)))
    2.11                  self.hvm_destroyPCIDevice(int(vslt, 16))
     3.1 --- a/tools/python/xen/xend/server/DevController.py	Mon Jul 14 10:10:14 2008 +0100
     3.2 +++ b/tools/python/xen/xend/server/DevController.py	Mon Jul 14 10:11:39 2008 +0100
     3.3 @@ -223,6 +223,12 @@ class DevController:
     3.4          raise VmError('%s devices may not be reconfigured' % self.deviceClass)
     3.5  
     3.6  
     3.7 +    def cleanupDeviceOnDomainDestroy(self, devid):
     3.8 +        """ Some devices may need special cleanup when the guest domain
     3.9 +            is destroyed.
    3.10 +        """
    3.11 +        return
    3.12 +
    3.13      def destroyDevice(self, devid, force):
    3.14          """Destroy the specified device.
    3.15  
    3.16 @@ -239,6 +245,8 @@ class DevController:
    3.17  
    3.18          dev = self.convertToDeviceNumber(devid)
    3.19  
    3.20 +        self.cleanupDeviceOnDomainDestroy(dev)
    3.21 +
    3.22          # Modify online status /before/ updating state (latter is watched by
    3.23          # drivers, so this ordering avoids a race).
    3.24          self.writeBackend(dev, 'online', "0")
     4.1 --- a/tools/python/xen/xend/server/pciif.py	Mon Jul 14 10:10:14 2008 +0100
     4.2 +++ b/tools/python/xen/xend/server/pciif.py	Mon Jul 14 10:11:39 2008 +0100
     4.3 @@ -28,7 +28,7 @@ from xen.xend.server.DevController impor
     4.4  
     4.5  import xen.lowlevel.xc
     4.6  
     4.7 -from xen.util.pci import PciDevice
     4.8 +from xen.util.pci import *
     4.9  import resource
    4.10  import re
    4.11  
    4.12 @@ -74,7 +74,7 @@ class PciController(DevController):
    4.13              if vslt is not None:
    4.14                  vslots = vslots + vslt + ";"
    4.15  
    4.16 -            back['dev-%i' % pcidevid] = "%04x:%02x:%02x.%02x" % \
    4.17 +            back['dev-%i' % pcidevid] = "%04x:%02x:%02x.%01x" % \
    4.18                                          (domain, bus, slot, func)
    4.19              back['uuid-%i' % pcidevid] = pci_config.get('uuid', '')
    4.20              pcidevid += 1
    4.21 @@ -284,8 +284,13 @@ class PciController(DevController):
    4.22                      "bind your slot/device to the PCI backend using sysfs" \
    4.23                      )%(dev.name))
    4.24  
    4.25 +        if dev.has_non_page_aligned_bar:
    4.26 +            raise VmError("pci: %: non-page-aligned MMIO BAR found." % dev.name)
    4.27 +
    4.28          self.CheckSiblingDevices(fe_domid, dev)
    4.29  
    4.30 +        dev.do_FLR()
    4.31 +
    4.32          PCIQuirk(dev.vendor, dev.device, dev.subvendor, dev.subdevice, domain, 
    4.33                  bus, slot, func)
    4.34  
    4.35 @@ -352,11 +357,48 @@ class PciController(DevController):
    4.36      def setupDevice(self, config):
    4.37          """Setup devices from config
    4.38          """
    4.39 +        pci_str_list = []
    4.40 +        pci_dev_list = []
    4.41          for pci_config in config.get('devs', []):
    4.42              domain = parse_hex(pci_config.get('domain', 0))
    4.43              bus = parse_hex(pci_config.get('bus', 0))
    4.44              slot = parse_hex(pci_config.get('slot', 0))
    4.45              func = parse_hex(pci_config.get('func', 0))            
    4.46 +            pci_str = '%04x:%02x:%02x.%01x' % (domain, bus, slot, func)
    4.47 +            pci_str_list = pci_str_list + [pci_str]
    4.48 +            pci_dev_list = pci_dev_list + [(domain, bus, slot, func)]
    4.49 +
    4.50 +        for (domain, bus, slot, func) in pci_dev_list:
    4.51 +            try:
    4.52 +                dev = PciDevice(domain, bus, slot, func)
    4.53 +            except Exception, e:
    4.54 +                raise VmError("pci: failed to locate device and "+
    4.55 +                        "parse it's resources - "+str(e))
    4.56 +            if (dev.dev_type == DEV_TYPE_PCIe_ENDPOINT) and not dev.pcie_flr:
    4.57 +                funcs = dev.find_all_the_multi_functions()
    4.58 +                for f in funcs:
    4.59 +                    if not f in pci_str_list:
    4.60 +                        err_msg = 'pci: % must be co-assigned to guest with %s'
    4.61 +                        raise VmError(err_msg % (f, dev.name))
    4.62 +            elif dev.dev_type == DEV_TYPE_PCI:
    4.63 +                if dev.bus == 0:
    4.64 +                    if not dev.pci_af_flr:
    4.65 +                        err_msg = 'pci: %s is not assignable: it is on ' + \
    4.66 +                            'bus 0,  but lacks of FLR capability'
    4.67 +                        raise VmError(err_msg % dev.name)
    4.68 +                else:
    4.69 +                    # All devices behind the uppermost PCI/PCI-X bridge must be\
    4.70 +                    # co-assigned to the same guest.
    4.71 +                    devs_str = dev.find_coassigned_devices(True)
    4.72 +                    # Remove the element 0 which is a bridge
    4.73 +                    del devs_str[0]
    4.74 +
    4.75 +                    for s in devs_str:
    4.76 +                        if not s in pci_str_list:
    4.77 +                            err_msg = 'pci: %s must be co-assigned to guest with %s'
    4.78 +                            raise VmError(err_msg % (s, dev.name))
    4.79 +
    4.80 +        for (domain, bus, slot, func) in pci_dev_list:
    4.81              self.setupOneDevice(domain, bus, slot, func)
    4.82  
    4.83          return
    4.84 @@ -419,6 +461,7 @@ class PciController(DevController):
    4.85              if rc<0:
    4.86                  raise VmError(('pci: failed to configure irq on device '+
    4.87                              '%s - errno=%d')%(dev.name,rc))
    4.88 +        dev.do_FLR()
    4.89  
    4.90      def cleanupDevice(self, devid):
    4.91          """ Detach I/O resources for device and cleanup xenstore nodes
    4.92 @@ -471,6 +514,22 @@ class PciController(DevController):
    4.93  
    4.94          return new_num_devs
    4.95  
    4.96 +    def cleanupDeviceOnDomainDestroy(self, devid):
    4.97 +        num_devs = int(self.readBackend(devid, 'num_devs'))
    4.98 +        dev_str_list = []
    4.99 +        for i in range(num_devs):
   4.100 +            dev_str = self.readBackend(devid, 'dev-%i' % i)
   4.101 +            dev_str_list = dev_str_list + [dev_str]
   4.102 +
   4.103 +        for dev_str in dev_str_list:
   4.104 +            (dom, b, d, f) = parse_pci_name(dev_str)
   4.105 +            try:
   4.106 +                dev = PciDevice(dom, b, d, f)
   4.107 +            except Exception, e:
   4.108 +                raise VmError("pci: failed to locate device and "+
   4.109 +                        "parse it's resources - "+str(e))
   4.110 +            dev.do_FLR()
   4.111 +
   4.112      def waitForBackend(self,devid):
   4.113          return (0, "ok - no hotplug")
   4.114