ia64/xen-unstable

changeset 18374:c5a7ceb199cd

vtd: do FLR before xc.domain_destroy()

When we "xm destroy" a guest with assigned devices, in
xen/common/domain.c:domain_kill(), we relinquish guest's memory first,
then invoke domain_destroy() -> complete_domain_destroy() ->
arch_domain_destroy() -> pci_release_devices()/iommu_domain_destroy().

Between relinquish_memory() and domain_destroy(), a device may be
issuing DMA request and access guest's relinquished memory.

We can do_FLR() to quiesce devices before xc.domain_destroy().

Signed-off-by: Dexuan Cui <dexuan.cui@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Tue Aug 26 14:11:39 2008 +0100 (2008-08-26)
parents 493a0a87919e
children c2472ded5c7c
files tools/python/xen/util/pci.py tools/python/xen/xend/XendDomain.py tools/python/xen/xend/XendDomainInfo.py tools/python/xen/xend/server/DevController.py tools/python/xen/xend/server/pciif.py
line diff
     1.1 --- a/tools/python/xen/util/pci.py	Tue Aug 26 14:10:17 2008 +0100
     1.2 +++ b/tools/python/xen/util/pci.py	Tue Aug 26 14:11:39 2008 +0100
     1.3 @@ -40,6 +40,7 @@ DEV_TYPE_PCIe_BRIDGE    = 1
     1.4  DEV_TYPE_PCI_BRIDGE     = 2
     1.5  DEV_TYPE_PCI            = 3    
     1.6  
     1.7 +PCI_VENDOR_ID = 0x0
     1.8  PCI_STATUS = 0x6
     1.9  PCI_CLASS_DEVICE = 0x0a
    1.10  PCI_CLASS_BRIDGE_PCI = 0x0604
    1.11 @@ -69,6 +70,11 @@ PCI_PM_CTRL_NO_SOFT_RESET = 0x0004
    1.12  PCI_PM_CTRL_STATE_MASK = 0x0003
    1.13  PCI_D3hot = 3
    1.14  
    1.15 +VENDOR_INTEL  = 0x8086
    1.16 +PCI_CAP_ID_VENDOR_SPECIFIC_CAP = 0x09
    1.17 +PCI_CLASS_ID_USB = 0x0c03
    1.18 +PCI_USB_FLRCTRL = 0x4
    1.19 +
    1.20  PCI_CAP_ID_AF = 0x13
    1.21  PCI_AF_CAPs   = 0x3
    1.22  PCI_AF_CAPs_TP_FLR = 0x3
    1.23 @@ -487,7 +493,7 @@ class PciDevice:
    1.24      def do_Dstate_transition(self):
    1.25          pos = self.find_cap_offset(PCI_CAP_ID_PM)
    1.26          if pos == 0:
    1.27 -            return 
    1.28 +            return False
    1.29          
    1.30          (pci_list, cfg_list) = save_pci_conf_space([self.name])
    1.31          
    1.32 @@ -504,6 +510,31 @@ class PciDevice:
    1.33          time.sleep(0.010)
    1.34  
    1.35          restore_pci_conf_space((pci_list, cfg_list))
    1.36 +        return True
    1.37 +
    1.38 +    def do_vendor_specific_FLR_method(self):
    1.39 +        pos = self.find_cap_offset(PCI_CAP_ID_VENDOR_SPECIFIC_CAP)
    1.40 +        if pos == 0:
    1.41 +            return
    1.42 +
    1.43 +        vendor_id = self.pci_conf_read16(PCI_VENDOR_ID)
    1.44 +        if vendor_id != VENDOR_INTEL:
    1.45 +            return
    1.46 +
    1.47 +        class_id = self.pci_conf_read16(PCI_CLASS_DEVICE)
    1.48 +        if class_id != PCI_CLASS_ID_USB:
    1.49 +            return
    1.50 +
    1.51 +        (pci_list, cfg_list) = save_pci_conf_space([self.name])
    1.52 +
    1.53 +        self.pci_conf_write8(pos + PCI_USB_FLRCTRL, 1)
    1.54 +        time.sleep(0.010)
    1.55 +
    1.56 +        restore_pci_conf_space((pci_list, cfg_list))
    1.57 +
    1.58 +    def do_FLR_for_integrated_device(self):
    1.59 +        if not self.do_Dstate_transition():
    1.60 +            self.do_vendor_specific_FLR_method()
    1.61  
    1.62      def find_all_the_multi_functions(self):
    1.63          sysfs_mnt = find_sysfs_mnt()
    1.64 @@ -676,7 +707,7 @@ class PciDevice:
    1.65                  restore_pci_conf_space((pci_list, cfg_list))
    1.66              else:
    1.67                  if self.bus == 0:
    1.68 -                    self.do_Dstate_transition()
    1.69 +                    self.do_FLR_for_integrated_device()
    1.70                  else:
    1.71                      funcs = self.find_all_the_multi_functions()
    1.72                      self.devs_check_driver(funcs)
    1.73 @@ -697,7 +728,7 @@ class PciDevice:
    1.74                  restore_pci_conf_space((pci_list, cfg_list))
    1.75              else:
    1.76                  if self.bus == 0:
    1.77 -                    self.do_Dstate_transition()
    1.78 +                    self.do_FLR_for_integrated_device()
    1.79                  else:
    1.80                      devs = self.find_coassigned_devices(False)
    1.81                      # Remove the element 0 which is a bridge
     2.1 --- a/tools/python/xen/xend/XendDomain.py	Tue Aug 26 14:10:17 2008 +0100
     2.2 +++ b/tools/python/xen/xend/XendDomain.py	Tue Aug 26 14:11:39 2008 +0100
     2.3 @@ -419,6 +419,8 @@ class XendDomain:
     2.4                  except VmError:
     2.5                      log.exception("Unable to recreate domain")
     2.6                      try:
     2.7 +                        xc.domain_pause(domid)
     2.8 +                        do_FLR(domid)
     2.9                          xc.domain_destroy(domid)
    2.10                      except:
    2.11                          log.exception("Hard destruction of domain failed: %d" %
    2.12 @@ -1255,6 +1257,8 @@ class XendDomain:
    2.13              val = dominfo.destroy()
    2.14          else:
    2.15              try:
    2.16 +                xc.domain_pause(int(domid))
    2.17 +                do_FLR(int(domid))
    2.18                  val = xc.domain_destroy(int(domid))
    2.19              except ValueError:
    2.20                  raise XendInvalidDomain(domid)
     3.1 --- a/tools/python/xen/xend/XendDomainInfo.py	Tue Aug 26 14:10:17 2008 +0100
     3.2 +++ b/tools/python/xen/xend/XendDomainInfo.py	Tue Aug 26 14:11:39 2008 +0100
     3.3 @@ -287,6 +287,28 @@ def dom_get(dom):
     3.4          log.trace("domain_getinfo(%d) failed, ignoring: %s", dom, str(err))
     3.5      return None
     3.6  
     3.7 +def do_FLR(domid):
     3.8 +    from xen.xend.server.pciif import parse_pci_name, PciDevice
     3.9 +    path = '/local/domain/0/backend/pci/%u/0/' % domid
    3.10 +    num_devs = xstransact.Read(path + 'num_devs');
    3.11 +    if num_devs is None or num_devs == "":
    3.12 +        return;
    3.13 +
    3.14 +    num_devs = int(xstransact.Read(path + 'num_devs'));
    3.15 +
    3.16 +    dev_str_list = []
    3.17 +    for i in range(num_devs):
    3.18 +        dev_str = xstransact.Read(path + 'dev-%i' % i)
    3.19 +        dev_str_list = dev_str_list + [dev_str]
    3.20 +
    3.21 +    for dev_str in dev_str_list:
    3.22 +        (dom, b, d, f) = parse_pci_name(dev_str)
    3.23 +        try:
    3.24 +            dev = PciDevice(dom, b, d, f)
    3.25 +        except Exception, e:
    3.26 +            raise VmError("pci: failed to locate device and "+
    3.27 +                    "parse it's resources - "+str(e))
    3.28 +        dev.do_FLR()
    3.29  
    3.30  class XendDomainInfo:
    3.31      """An object represents a domain.
    3.32 @@ -2410,6 +2432,8 @@ class XendDomainInfo:
    3.33          try:
    3.34              if self.domid is not None:
    3.35                  xc.domain_destroy_hook(self.domid)
    3.36 +                xc.domain_pause(self.domid)
    3.37 +                do_FLR(self.domid)
    3.38                  xc.domain_destroy(self.domid)
    3.39                  for state in DOM_STATES_OLD:
    3.40                      self.info[state] = 0
     4.1 --- a/tools/python/xen/xend/server/DevController.py	Tue Aug 26 14:10:17 2008 +0100
     4.2 +++ b/tools/python/xen/xend/server/DevController.py	Tue Aug 26 14:11:39 2008 +0100
     4.3 @@ -223,12 +223,6 @@ class DevController:
     4.4          raise VmError('%s devices may not be reconfigured' % self.deviceClass)
     4.5  
     4.6  
     4.7 -    def cleanupDeviceOnDomainDestroy(self, devid):
     4.8 -        """ Some devices may need special cleanup when the guest domain
     4.9 -            is destroyed.
    4.10 -        """
    4.11 -        return
    4.12 -
    4.13      def destroyDevice(self, devid, force):
    4.14          """Destroy the specified device.
    4.15  
    4.16 @@ -245,8 +239,6 @@ class DevController:
    4.17  
    4.18          dev = self.convertToDeviceNumber(devid)
    4.19  
    4.20 -        self.cleanupDeviceOnDomainDestroy(dev)
    4.21 -
    4.22          # Modify online status /before/ updating state (latter is watched by
    4.23          # drivers, so this ordering avoids a race).
    4.24          self.writeBackend(dev, 'online', "0")
     5.1 --- a/tools/python/xen/xend/server/pciif.py	Tue Aug 26 14:10:17 2008 +0100
     5.2 +++ b/tools/python/xen/xend/server/pciif.py	Tue Aug 26 14:11:39 2008 +0100
     5.3 @@ -383,10 +383,10 @@ class PciController(DevController):
     5.4              if (dev.dev_type == DEV_TYPE_PCIe_ENDPOINT) and not dev.pcie_flr:
     5.5                  if dev.bus == 0:
     5.6                      # We cope with this case by using the Dstate transition
     5.7 -                    # method for now.
     5.8 +                    # method or some vendor specific methods for now.
     5.9                      err_msg = 'pci: %s: it is on bus 0, but has no PCIe' +\
    5.10                          ' FLR Capability. Will try the Dstate transition'+\
    5.11 -                        ' method if available.'
    5.12 +                        ' method or some vendor specific methods if available.'
    5.13                      log.warn(err_msg % dev.name)
    5.14                  else:
    5.15                      funcs = dev.find_all_the_multi_functions()
    5.16 @@ -404,10 +404,11 @@ class PciController(DevController):
    5.17                  if dev.bus == 0 or arch.type == "ia64":
    5.18                      if not dev.pci_af_flr:
    5.19                          # We cope with this case by using the Dstate transition
    5.20 -                        # method for now.
    5.21 +                        # method or some vendor specific methods for now.
    5.22                          err_msg = 'pci: %s: it is on bus 0, but has no PCI' +\
    5.23                              ' Advanced Capabilities for FLR. Will try the'+\
    5.24 -                            ' Dstate transition method if available.'
    5.25 +                            ' Dstate transition method or some vendor' +\
    5.26 +                            ' specific methods if available.'
    5.27                          log.warn(err_msg % dev.name)
    5.28                  else:
    5.29                      # All devices behind the uppermost PCI/PCI-X bridge must be\
    5.30 @@ -543,22 +544,6 @@ class PciController(DevController):
    5.31  
    5.32          return new_num_devs
    5.33  
    5.34 -    def cleanupDeviceOnDomainDestroy(self, devid):
    5.35 -        num_devs = int(self.readBackend(devid, 'num_devs'))
    5.36 -        dev_str_list = []
    5.37 -        for i in range(num_devs):
    5.38 -            dev_str = self.readBackend(devid, 'dev-%i' % i)
    5.39 -            dev_str_list = dev_str_list + [dev_str]
    5.40 -
    5.41 -        for dev_str in dev_str_list:
    5.42 -            (dom, b, d, f) = parse_pci_name(dev_str)
    5.43 -            try:
    5.44 -                dev = PciDevice(dom, b, d, f)
    5.45 -            except Exception, e:
    5.46 -                raise VmError("pci: failed to locate device and "+
    5.47 -                        "parse it's resources - "+str(e))
    5.48 -            dev.do_FLR()
    5.49 -
    5.50      def waitForBackend(self,devid):
    5.51          return (0, "ok - no hotplug")
    5.52