ia64/xen-unstable

view tools/python/xen/xend/server/pciif.py @ 19722:691087b8d4ac

xend: pass-through: Use AUTO_PHP_SLOT as unknown vslot

This fixes a few cases where 0 is still used for an known vslot.

Signed-off-by: Simon Horman <horms@verge.net.au>
author Keir Fraser <keir.fraser@citrix.com>
date Thu Jun 04 10:43:20 2009 +0100 (2009-06-04)
parents 19fc31fe2e31
children a5f584c1e2f6
line source
1 #============================================================================
2 # This library is free software; you can redistribute it and/or
3 # modify it under the terms of version 2.1 of the GNU Lesser General Public
4 # License as published by the Free Software Foundation.
5 #
6 # This library is distributed in the hope that it will be useful,
7 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
9 # Lesser General Public License for more details.
10 #
11 # You should have received a copy of the GNU Lesser General Public
12 # License along with this library; if not, write to the Free Software
13 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
14 #============================================================================
15 # Copyright (C) 2004, 2005 Mike Wray <mike.wray@hp.com>
16 # Copyright (C) 2005 XenSource Ltd
17 #============================================================================
20 import types
21 import time
23 from xen.xend import sxp
24 from xen.xend import arch
25 from xen.xend.XendError import VmError
26 from xen.xend.XendLogging import log
27 from xen.xend.XendConstants import *
29 from xen.xend.server.DevController import DevController
30 from xen.xend.server.DevConstants import xenbusState
32 import xen.lowlevel.xc
34 from xen.util.pci import *
35 import resource
36 import re
38 from xen.xend.server.pciquirk import *
39 from xen.xend.xenstore.xstransact import xstransact
40 from xen.xend.xenstore.xswatch import xswatch
42 xc = xen.lowlevel.xc.xc()
44 #Calculate PAGE_SHIFT: number of bits to shift an address to get the page number
45 PAGE_SIZE = resource.getpagesize()
46 PAGE_SHIFT = 0
47 t = PAGE_SIZE
48 while not (t&1):
49 t>>=1
50 PAGE_SHIFT+=1
52 def parse_hex(val):
53 try:
54 if isinstance(val, types.StringTypes):
55 return int(val, 16)
56 else:
57 return val
58 except ValueError:
59 return None
61 class PciController(DevController):
63 def __init__(self, vm):
64 self.aerStateWatch = None
65 DevController.__init__(self, vm)
68 def getDeviceDetails(self, config):
69 """@see DevController.getDeviceDetails"""
70 back = {}
71 pcidevid = 0
72 for pci_config in config.get('devs', []):
73 domain = parse_hex(pci_config.get('domain', 0))
74 bus = parse_hex(pci_config.get('bus', 0))
75 slot = parse_hex(pci_config.get('slot', 0))
76 func = parse_hex(pci_config.get('func', 0))
77 vslot = parse_hex(pci_config.get('vslot',
78 '0x' + AUTO_PHP_SLOT_STR))
80 if pci_config.has_key('opts'):
81 opts = serialise_pci_opts(pci_config['opts'])
82 back['opts-%i' % pcidevid] = opts
84 back['dev-%i' % pcidevid] = "%04x:%02x:%02x.%01x" % \
85 (domain, bus, slot, func)
86 back['uuid-%i' % pcidevid] = pci_config.get('uuid', '')
87 back['vslot-%i' % pcidevid] = "%02x" % vslot
88 pcidevid += 1
90 back['num_devs']=str(pcidevid)
91 back['uuid'] = config.get('uuid','')
92 if 'pci_msitranslate' in self.vm.info['platform']:
93 back['msitranslate']=str(self.vm.info['platform']['pci_msitranslate'])
94 if 'pci_power_mgmt' in self.vm.info['platform']:
95 back['power_mgmt']=str(self.vm.info['platform']['pci_power_mgmt'])
97 return (0, back, {})
99 def reconfigureDevice_find(self, devid, nsearch_dev, match_dev):
100 for j in range(nsearch_dev):
101 if match_dev == self.readBackend(devid, 'dev-%i' % j):
102 return j
103 return None
105 def reconfigureDevice(self, _, config):
106 """@see DevController.reconfigureDevice"""
107 (devid, back, front) = self.getDeviceDetails(config)
108 num_devs = int(back['num_devs'])
109 states = config.get('states', [])
110 num_olddevs = int(self.readBackend(devid, 'num_devs'))
112 for i in range(num_devs):
113 try:
114 dev = back['dev-%i' % i]
115 state = states[i]
116 uuid = back['uuid-%i' %i]
117 opts = ''
118 if 'opts-%i' % i in back:
119 opts = back['opts-%i' % i]
120 except:
121 raise XendError('Error reading config')
123 if state == 'Initialising':
124 devno = self.reconfigureDevice_find(devid, num_olddevs, dev)
125 if devno == None:
126 devno = num_olddevs + i
127 log.debug('Attaching PCI device %s.' % dev)
128 attaching = True
129 else:
130 log.debug('Reconfiguring PCI device %s.' % dev)
131 attaching = False
133 (domain, bus, slotfunc) = dev.split(':')
134 (slot, func) = slotfunc.split('.')
135 domain = parse_hex(domain)
136 bus = parse_hex(bus)
137 slot = parse_hex(slot)
138 func = parse_hex(func)
139 self.setupOneDevice(domain, bus, slot, func)
141 self.writeBackend(devid, 'dev-%i' % devno, dev)
142 self.writeBackend(devid, 'state-%i' % devno,
143 str(xenbusState['Initialising']))
144 self.writeBackend(devid, 'uuid-%i' % devno, uuid)
145 if len(opts) > 0:
146 self.writeBackend(devid, 'opts-%i' % devno, opts)
147 if back.has_key('vslot-%i' % i):
148 self.writeBackend(devid, 'vslot-%i' % devno,
149 back['vslot-%i' % i])
151 # If a device is being attached then num_devs will grow
152 if attaching:
153 self.writeBackend(devid, 'num_devs', str(devno + 1))
155 elif state == 'Closing':
156 # PCI device detachment
157 devno = self.reconfigureDevice_find(devid, num_olddevs, dev)
158 if devno == None:
159 raise XendError('Device %s is not connected' % dev)
160 log.debug('Detaching device %s' % dev)
161 self.writeBackend(devid, 'state-%i' % devno,
162 str(xenbusState['Closing']))
164 else:
165 raise XendError('Error configuring device %s: invalid state %s'
166 % (dev,state))
168 self.writeBackend(devid, 'state', str(xenbusState['Reconfiguring']))
170 return self.readBackend(devid, 'uuid')
173 def getDeviceConfiguration(self, devid, transaction = None):
174 result = DevController.getDeviceConfiguration(self, devid, transaction)
175 num_devs = self.readBackend(devid, 'num_devs')
176 pci_devs = []
178 for i in range(int(num_devs)):
179 dev_config = self.readBackend(devid, 'dev-%d' % i)
181 pci_match = re.match(r"((?P<domain>[0-9a-fA-F]{1,4})[:,])?" +
182 r"(?P<bus>[0-9a-fA-F]{1,2})[:,]" +
183 r"(?P<slot>[0-9a-fA-F]{1,2})[.,]" +
184 r"(?P<func>[0-7]{1,2})$", dev_config)
186 if pci_match!=None:
187 pci_dev_info = pci_match.groupdict()
188 dev_dict = {'domain': '0x%(domain)s' % pci_dev_info,
189 'bus': '0x%(bus)s' % pci_dev_info,
190 'slot': '0x%(slot)s' % pci_dev_info,
191 'func': '0x%(func)s' % pci_dev_info}
193 # Per device uuid info
194 dev_dict['uuid'] = self.readBackend(devid, 'uuid-%d' % i)
195 dev_dict['vslot'] = '0x%s' % \
196 self.readBackend(devid, 'vslot-%d' % i)
198 #append opts info
199 opts = self.readBackend(devid, 'opts-%d' % i)
200 if opts is not None:
201 dev_dict['opts'] = opts
203 pci_devs.append(dev_dict)
205 result['devs'] = pci_devs
206 result['uuid'] = self.readBackend(devid, 'uuid')
207 return result
209 def configuration(self, devid, transaction = None):
210 """Returns SXPR for devices on domain.
212 @note: we treat this dict especially to convert to
213 SXP because it is not a straight dict of strings."""
215 configDict = self.getDeviceConfiguration(devid, transaction)
216 sxpr = [self.deviceClass]
218 # remove devs
219 devs = configDict.pop('devs', [])
221 for dev in devs:
222 dev_sxpr = ['dev']
223 for dev_key, dev_val in dev.items():
224 if dev_key == 'opts':
225 dev_sxpr.append(['opts', split_pci_opts(dev_val)])
226 else:
227 dev_sxpr.append([dev_key, dev_val])
228 sxpr.append(dev_sxpr)
230 for key, val in configDict.items():
231 if type(val) == type(list()):
232 for v in val:
233 sxpr.append([key, v])
234 else:
235 sxpr.append([key, val])
237 return sxpr
239 def CheckSiblingDevices(self, domid, dev):
240 """ Check if all sibling devices of dev are owned by pciback
241 """
242 if not self.vm.info.is_hvm():
243 return
245 group_str = xc.get_device_group(domid, dev.domain, dev.bus, dev.slot, dev.func)
246 if group_str == "":
247 return
249 #group string format xx:xx.x,xx:xx.x,
250 devstr_len = group_str.find(',')
251 for i in range(0, len(group_str), devstr_len + 1):
252 (bus, slotfunc) = group_str[i:i + devstr_len].split(':')
253 (slot, func) = slotfunc.split('.')
254 b = parse_hex(bus)
255 d = parse_hex(slot)
256 f = parse_hex(func)
257 try:
258 sdev = PciDevice(dev.domain, b, d, f)
259 except Exception, e:
260 #no dom0 drivers bound to sdev
261 continue
263 if sdev.driver!='pciback':
264 raise VmError(("pci: PCI Backend does not own\n "+ \
265 "sibling device %s of device %s\n"+ \
266 "See the pciback.hide kernel "+ \
267 "command-line parameter or\n"+ \
268 "bind your slot/device to the PCI backend using sysfs" \
269 )%(sdev.name, dev.name))
270 return
272 def setupOneDevice(self, domain, bus, slot, func):
273 """ Attach I/O resources for device to frontend domain
274 """
275 fe_domid = self.getDomid()
277 try:
278 dev = PciDevice(domain, bus, slot, func)
279 except Exception, e:
280 raise VmError("pci: failed to locate device and "+
281 "parse it's resources - "+str(e))
283 if dev.driver!='pciback':
284 raise VmError(("pci: PCI Backend does not own device "+ \
285 "%s\n"+ \
286 "See the pciback.hide kernel "+ \
287 "command-line parameter or\n"+ \
288 "bind your slot/device to the PCI backend using sysfs" \
289 )%(dev.name))
291 if dev.has_non_page_aligned_bar and arch.type != "ia64":
292 raise VmError("pci: %s: non-page-aligned MMIO BAR found." % dev.name)
294 self.CheckSiblingDevices(fe_domid, dev)
296 # We don't do FLR when we create domain and hotplug device into guest,
297 # namely, we only do FLR when we destroy domain or hotplug device from
298 # guest. This is mainly to work around the race condition in hotplug code
299 # paths. See the changeset's description for details.
300 # if arch.type != "ia64":
301 # dev.do_FLR()
303 PCIQuirk(dev.vendor, dev.device, dev.subvendor, dev.subdevice, domain,
304 bus, slot, func)
306 if not self.vm.info.is_hvm():
307 # Setup IOMMU device assignment
308 pci_str = "0x%x, 0x%x, 0x%x, 0x%x" % (domain, bus, slot, func)
309 bdf = xc.assign_device(fe_domid, pci_str)
310 if bdf > 0:
311 raise VmError("Failed to assign device to IOMMU (%x:%x.%x)"
312 % (bus, slot, func))
313 log.debug("pci: assign device %x:%x.%x" % (bus, slot, func))
315 for (start, size) in dev.ioports:
316 log.debug('pci: enabling ioport 0x%x/0x%x'%(start,size))
317 rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start,
318 nr_ports = size, allow_access = True)
319 if rc<0:
320 raise VmError(('pci: failed to configure I/O ports on device '+
321 '%s - errno=%d')%(dev.name,rc))
323 for (start, size) in dev.iomem:
324 # Convert start/size from bytes to page frame sizes
325 start_pfn = start>>PAGE_SHIFT
326 # Round number of pages up to nearest page boundary (if not on one)
327 nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT
329 log.debug('pci: enabling iomem 0x%x/0x%x pfn 0x%x/0x%x'% \
330 (start,size,start_pfn,nr_pfns))
331 rc = xc.domain_iomem_permission(domid = fe_domid,
332 first_pfn = start_pfn,
333 nr_pfns = nr_pfns,
334 allow_access = True)
335 if rc<0:
336 raise VmError(('pci: failed to configure I/O memory on device '+
337 '%s - errno=%d')%(dev.name,rc))
339 if dev.msix:
340 for (start, size) in dev.msix_iomem:
341 start_pfn = start>>PAGE_SHIFT
342 nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT
343 log.debug('pci-msix: remove permission for 0x%x/0x%x 0x%x/0x%x' % \
344 (start,size, start_pfn, nr_pfns))
345 rc = xc.domain_iomem_permission(domid = fe_domid,
346 first_pfn = start_pfn,
347 nr_pfns = nr_pfns,
348 allow_access = False)
349 if rc<0:
350 raise VmError(('pci: failed to remove msi-x iomem'))
352 rc = xc.physdev_map_pirq(domid = fe_domid,
353 index = dev.irq,
354 pirq = dev.irq)
355 if rc < 0:
356 raise VmError(('pci: failed to map irq on device '+
357 '%s - errno=%d')%(dev.name,rc))
358 if dev.irq>0:
359 log.debug('pci: enabling irq %d'%dev.irq)
360 rc = xc.domain_irq_permission(domid = fe_domid, pirq = dev.irq,
361 allow_access = True)
362 if rc<0:
363 raise VmError(('pci: failed to configure irq on device '+
364 '%s - errno=%d')%(dev.name,rc))
366 def setupDevice(self, config):
367 """Setup devices from config
368 """
369 pci_str_list = []
370 pci_dev_list = []
371 for pci_config in config.get('devs', []):
372 domain = parse_hex(pci_config.get('domain', 0))
373 bus = parse_hex(pci_config.get('bus', 0))
374 slot = parse_hex(pci_config.get('slot', 0))
375 func = parse_hex(pci_config.get('func', 0))
376 pci_str = '%04x:%02x:%02x.%01x' % (domain, bus, slot, func)
377 pci_str_list = pci_str_list + [pci_str]
378 pci_dev_list = pci_dev_list + [(domain, bus, slot, func)]
380 if len(pci_str_list) != len(set(pci_str_list)):
381 raise VmError('pci: duplicate devices specified in guest config?')
383 for (domain, bus, slot, func) in pci_dev_list:
384 try:
385 dev = PciDevice(domain, bus, slot, func)
386 except Exception, e:
387 raise VmError("pci: failed to locate device and "+
388 "parse it's resources - "+str(e))
389 if (dev.dev_type == DEV_TYPE_PCIe_ENDPOINT) and not dev.pcie_flr:
390 if dev.bus == 0:
391 # We cope with this case by using the Dstate transition
392 # method or some vendor specific methods for now.
393 err_msg = 'pci: %s: it is on bus 0, but has no PCIe' +\
394 ' FLR Capability. Will try the Dstate transition'+\
395 ' method or some vendor specific methods if available.'
396 log.warn(err_msg % dev.name)
397 else:
398 funcs = dev.find_all_the_multi_functions()
399 dev.devs_check_driver(funcs)
400 for f in funcs:
401 if not f in pci_str_list:
402 (f_dom, f_bus, f_slot, f_func) = parse_pci_name(f)
403 f_pci_str = '0x%x,0x%x,0x%x,0x%x' % \
404 (f_dom, f_bus, f_slot, f_func)
405 # f has been assigned to other guest?
406 if xc.test_assign_device(0, f_pci_str) != 0:
407 err_msg = 'pci: %s must be co-assigned to' + \
408 ' the same guest with %s'
409 raise VmError(err_msg % (f, dev.name))
410 elif dev.dev_type == DEV_TYPE_PCI:
411 if dev.bus == 0 or arch.type == "ia64":
412 if not dev.pci_af_flr:
413 # We cope with this case by using the Dstate transition
414 # method or some vendor specific methods for now.
415 err_msg = 'pci: %s: it is on bus 0, but has no PCI' +\
416 ' Advanced Capabilities for FLR. Will try the'+\
417 ' Dstate transition method or some vendor' +\
418 ' specific methods if available.'
419 log.warn(err_msg % dev.name)
420 else:
421 # All devices behind the uppermost PCI/PCI-X bridge must be\
422 # co-assigned to the same guest.
423 devs_str = dev.find_coassigned_pci_devices(True)
424 # Remove the element 0 which is a bridge
425 del devs_str[0]
427 dev.devs_check_driver(devs_str)
428 for s in devs_str:
429 if not s in pci_str_list:
430 (s_dom, s_bus, s_slot, s_func) = parse_pci_name(s)
431 s_pci_str = '0x%x,0x%x,0x%x,0x%x' % \
432 (s_dom, s_bus, s_slot, s_func)
433 # s has been assigned to other guest?
434 if xc.test_assign_device(0, s_pci_str) != 0:
435 err_msg = 'pci: %s must be co-assigned to the'+\
436 ' same guest with %s'
437 raise VmError(err_msg % (s, dev.name))
439 for (domain, bus, slot, func) in pci_dev_list:
440 self.setupOneDevice(domain, bus, slot, func)
441 wPath = '/local/domain/0/backend/pci/%u/0/aerState' % (self.getDomid())
442 self.aerStateWatch = xswatch(wPath, self._handleAerStateWatch)
443 log.debug('pci: register aer watch %s', wPath)
444 return
446 def _handleAerStateWatch(self, _):
447 log.debug('XendDomainInfo.handleAerStateWatch')
448 if self.getDomid() == 0:
449 raise XendError('Domain 0 cannot be shutdown')
450 readPath = '/local/domain/0/backend/pci/%u/0/aerState' % (self.getDomid())
451 action = xstransact.Read(readPath)
452 if action and action=='aerfail':
453 log.debug('shutdown domain because of aer handle error')
454 self.vm.shutdown('poweroff')
455 return True
458 def cleanupOneDevice(self, domain, bus, slot, func):
459 """ Detach I/O resources for device from frontend domain
460 """
461 fe_domid = self.getDomid()
463 try:
464 dev = PciDevice(domain, bus, slot, func)
465 except Exception, e:
466 raise VmError("pci: failed to locate device and "+
467 "parse it's resources - "+str(e))
469 if dev.driver!='pciback':
470 raise VmError(("pci: PCI Backend does not own device "+ \
471 "%s\n"+ \
472 "See the pciback.hide kernel "+ \
473 "command-line parameter or\n"+ \
474 "bind your slot/device to the PCI backend using sysfs" \
475 )%(dev.name))
477 # Need to do FLR here before deassign device in order to terminate
478 # DMA transaction, etc
479 dev.do_FLR()
481 pci_str = "0x%x, 0x%x, 0x%x, 0x%x" % (domain, bus, slot, func)
482 bdf = xc.deassign_device(fe_domid, pci_str)
483 if bdf > 0:
484 raise VmError("Failed to deassign device from IOMMU (%x:%x.%x)"
485 % (bus, slot, func))
486 log.debug("pci: Deassign device %x:%x.%x" % (bus, slot, func))
488 for (start, size) in dev.ioports:
489 log.debug('pci: disabling ioport 0x%x/0x%x'%(start,size))
490 rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start,
491 nr_ports = size, allow_access = False)
492 if rc<0:
493 raise VmError(('pci: failed to configure I/O ports on device '+
494 '%s - errno=%d')%(dev.name,rc))
496 for (start, size) in dev.iomem:
497 # Convert start/size from bytes to page frame sizes
498 start_pfn = start>>PAGE_SHIFT
499 # Round number of pages up to nearest page boundary (if not on one)
500 nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT
502 log.debug('pci: disabling iomem 0x%x/0x%x pfn 0x%x/0x%x'% \
503 (start,size,start_pfn,nr_pfns))
504 rc = xc.domain_iomem_permission(domid = fe_domid,
505 first_pfn = start_pfn,
506 nr_pfns = nr_pfns,
507 allow_access = False)
508 if rc<0:
509 raise VmError(('pci: failed to configure I/O memory on device '+
510 '%s - errno=%d')%(dev.name,rc))
512 if dev.irq>0:
513 log.debug('pci: disabling irq %d'%dev.irq)
514 rc = xc.domain_irq_permission(domid = fe_domid, pirq = dev.irq,
515 allow_access = False)
516 if rc<0:
517 raise VmError(('pci: failed to configure irq on device '+
518 '%s - errno=%d')%(dev.name,rc))
520 def cleanupDevice(self, devid):
521 """ Detach I/O resources for device and cleanup xenstore nodes
522 after reconfigure.
524 @param devid: The device ID
525 @type devid: int
526 @return: Return the number of devices connected
527 @rtype: int
528 """
529 num_devs = int(self.readBackend(devid, 'num_devs'))
530 new_num_devs = 0
531 for i in range(num_devs):
532 state = int(self.readBackend(devid, 'state-%i' % i))
533 if state == xenbusState['Closing']:
534 # Detach I/O resources.
535 dev = self.readBackend(devid, 'dev-%i' % i)
536 (domain, bus, slotfunc) = dev.split(':')
537 (slot, func) = slotfunc.split('.')
538 domain = parse_hex(domain)
539 bus = parse_hex(bus)
540 slot = parse_hex(slot)
541 func = parse_hex(func)
542 # In HVM case, I/O resources are disabled in ioemu.
543 self.cleanupOneDevice(domain, bus, slot, func)
544 # Remove xenstore nodes.
545 self.removeBackend(devid, 'dev-%i' % i)
546 self.removeBackend(devid, 'vdev-%i' % i)
547 self.removeBackend(devid, 'state-%i' % i)
548 self.removeBackend(devid, 'uuid-%i' % i)
549 tmpopts = self.readBackend(devid, 'opts-%i' % i)
550 if tmpopts is not None:
551 self.removeBackend(devid, 'opts-%i' % i)
552 else:
553 if new_num_devs != i:
554 tmpdev = self.readBackend(devid, 'dev-%i' % i)
555 self.writeBackend(devid, 'dev-%i' % new_num_devs, tmpdev)
556 self.removeBackend(devid, 'dev-%i' % i)
557 tmpvdev = self.readBackend(devid, 'vdev-%i' % i)
558 if tmpvdev is not None:
559 self.writeBackend(devid, 'vdev-%i' % new_num_devs,
560 tmpvdev)
561 self.removeBackend(devid, 'vdev-%i' % i)
562 tmpstate = self.readBackend(devid, 'state-%i' % i)
563 self.writeBackend(devid, 'state-%i' % new_num_devs, tmpstate)
564 self.removeBackend(devid, 'state-%i' % i)
565 tmpuuid = self.readBackend(devid, 'uuid-%i' % i)
566 self.writeBackend(devid, 'uuid-%i' % new_num_devs, tmpuuid)
567 self.removeBackend(devid, 'uuid-%i' % i)
568 tmpopts = self.readBackend(devid, 'opts-%i' % i)
569 if tmpopts is not None:
570 self.removeBackend(devid, 'opts-%i' % i)
571 new_num_devs = new_num_devs + 1
573 self.writeBackend(devid, 'num_devs', str(new_num_devs))
575 return new_num_devs
577 def destroyDevice(self, devid, force):
578 DevController.destroyDevice(self, devid, True)
579 log.debug('pci: unregister aer watch')
580 self.unwatchAerState()
582 def unwatchAerState(self):
583 """Remove the watch on the domain's aerState node, if any."""
584 try:
585 try:
586 if self.aerStateWatch:
587 self.aerStateWatch.unwatch()
588 finally:
589 self.aerStateWatch = None
590 except:
591 log.exception("Unwatching aerState failed.")
593 def waitForBackend(self,devid):
594 return (0, "ok - no hotplug")
596 def migrate(self, config, network, dst, step, domName):
597 raise XendError('Migration not permitted with assigned PCI device.')