ia64/xen-unstable

changeset 1900:3548afc2425d

bitkeeper revision 1.1108.14.1 (4104cfa1bp3FkfR2vYJwebaywi6JAg)

Stop a domain restarting if it's too soon since the last one.
author mjw@wray-m-3.hpl.hp.com
date Mon Jul 26 09:32:17 2004 +0000 (2004-07-26)
parents fbcefe1a0285
children dad2f070529a
files tools/python/xen/xend/XendDomain.py tools/python/xen/xend/XendDomainInfo.py
line diff
     1.1 --- a/tools/python/xen/xend/XendDomain.py	Fri Jul 23 17:16:11 2004 +0000
     1.2 +++ b/tools/python/xen/xend/XendDomain.py	Mon Jul 26 09:32:17 2004 +0000
     1.3 @@ -130,9 +130,11 @@ class XendDomain:
     1.4      def initial_refresh(self):
     1.5          """Refresh initial domain info from domain_db.
     1.6          """
     1.7 -        #for d in self.domain_db.values(): print 'db dom=', d
     1.8 +            
     1.9 +        def cb_all_ok(val):
    1.10 +            self.refresh()
    1.11 +
    1.12          domlist = xc.domain_getinfo()
    1.13 -        #for d in domlist: print 'xc dom=', d
    1.14          doms = {}
    1.15          for d in domlist:
    1.16              domid = str(d['dom'])
    1.17 @@ -140,22 +142,13 @@ class XendDomain:
    1.18          dlist = []
    1.19          for config in self.domain_db.values():
    1.20              domid = str(sxp.child_value(config, 'id'))
    1.21 -            #print "dom=", domid, "config=", config
    1.22              if domid in doms:
    1.23 -                #print "dom=", domid, "new"
    1.24 -                deferred = self._new_domain(config, doms[domid])
    1.25 -                dlist.append(deferred)
    1.26 +                d_dom = self._new_domain(config, doms[domid])
    1.27 +                dlist.append(d_dom)
    1.28              else:
    1.29 -                #print "dom=", domid, "del"
    1.30                  self._delete_domain(domid)
    1.31 -        deferred = defer.DeferredList(dlist, fireOnOneErrback=1)
    1.32 -        def cbok(val):
    1.33 -            #print "doms:"
    1.34 -            #for d in self.domain.values(): print 'dom', d
    1.35 -            self.refresh()
    1.36 -            #print "XendDomain>initial_refresh> doms:"
    1.37 -            #for d in self.domain.values(): print 'dom', d
    1.38 -        deferred.addCallback(cbok)
    1.39 +        d_all = defer.DeferredList(dlist, fireOnOneErrback=1)
    1.40 +        d_all.addCallback(cb_all_ok)
    1.41  
    1.42      def sync(self):
    1.43          """Sync domain db to disk.
    1.44 @@ -179,10 +172,13 @@ class XendDomain:
    1.45          @param info:      domain info from xen
    1.46          @return: deferred
    1.47          """
    1.48 +        def cbok(dominfo):
    1.49 +            self.domain[dominfo.id] = dominfo
    1.50 +            if dominfo.restart_pending():
    1.51 +                self.domain_restart_add(dominfo)
    1.52 +        
    1.53          deferred = XendDomainInfo.vm_recreate(savedinfo, info)
    1.54 -        def fn(dominfo):
    1.55 -            self.domain[dominfo.id] = dominfo
    1.56 -        deferred.addCallback(fn)
    1.57 +        deferred.addCallback(cbok)
    1.58          return deferred
    1.59  
    1.60      def _add_domain(self, id, info, notify=1):
    1.61 @@ -250,9 +246,9 @@ class XendDomain:
    1.62              if id not in self.domain:
    1.63                  savedinfo = None
    1.64                  deferred = XendDomainInfo.vm_recreate(savedinfo, d)
    1.65 -                def fn(dominfo):
    1.66 +                def cbok(dominfo):
    1.67                      self._add_domain(dominfo.id, dominfo)
    1.68 -                deferred.addCallback(fn)
    1.69 +                deferred.addCallback(cbok)
    1.70          # Remove entries for domains that no longer exist.
    1.71          for d in self.domain.values():
    1.72              info = doms.get(d.id)
    1.73 @@ -313,11 +309,11 @@ class XendDomain:
    1.74          @param config: configuration
    1.75          @return: deferred
    1.76          """
    1.77 -        deferred = XendDomainInfo.vm_create(config)
    1.78 -        def fn(dominfo):
    1.79 +        def cbok(dominfo):
    1.80              self._add_domain(dominfo.id, dominfo)
    1.81              return dominfo
    1.82 -        deferred.addCallback(fn)
    1.83 +        deferred = XendDomainInfo.vm_create(config)
    1.84 +        deferred.addCallback(cbok)
    1.85          return deferred
    1.86  
    1.87      def domain_restart(self, dominfo):
    1.88 @@ -326,11 +322,12 @@ class XendDomain:
    1.89          @param dominfo: domain object
    1.90          @return: deferred
    1.91          """
    1.92 -        deferred = dominfo.restart()
    1.93 -        def fn(dominfo):
    1.94 +        def cbok(dominfo):
    1.95              self._add_domain(dominfo.id, dominfo)
    1.96              return dominfo
    1.97 -        deferred.addCallback(fn)
    1.98 +        log.info("Restarting domain: id=%s name=%s", dominfo.id, dominfo.name)
    1.99 +        deferred = dominfo.restart()
   1.100 +        deferred.addCallback(cbok)
   1.101          return deferred        
   1.102  
   1.103      def domain_configure(self, id, config):
   1.104 @@ -348,11 +345,11 @@ class XendDomain:
   1.105              raise XendError("Invalid domain: " + str(id))
   1.106          if dominfo.config:
   1.107              raise XendError("Domain already configured: " + str(id))
   1.108 -        def fn(dominfo):
   1.109 +        def cbok(dominfo):
   1.110              self._add_domain(dominfo.id, dominfo)
   1.111              return dominfo
   1.112          deferred = dominfo.construct(config)
   1.113 -        deferred.addCallback(fn)
   1.114 +        deferred.addCallback(cbok)
   1.115          return deferred
   1.116      
   1.117      def domain_restore(self, src, progress=0):
   1.118 @@ -363,11 +360,11 @@ class XendDomain:
   1.119          @return: deferred
   1.120          """
   1.121          
   1.122 -        def fn(dominfo):
   1.123 +        def cbok(dominfo):
   1.124              self._add_domain(dominfo.id, dominfo)
   1.125              return dominfo
   1.126          deferred = XendDomainInfo.vm_restore(src, progress=progress)
   1.127 -        deferred.addCallback(fn)
   1.128 +        deferred.addCallback(cbok)
   1.129          return deferred
   1.130      
   1.131      def domain_get(self, id):
   1.132 @@ -439,9 +436,12 @@ class XendDomain:
   1.133          restart = (force and reason == 'reboot') or dominfo.restart_needed(reason)
   1.134          if restart:
   1.135              dominfo.restarting()
   1.136 -            self.restarts[id] = dominfo
   1.137 -            log.info('Scheduling restart for domain: id=%s name=%s', id, dominfo.name)
   1.138 -            self.domain_restarts_schedule()
   1.139 +            self.domain_restart_add(dominfo)
   1.140 +
   1.141 +    def domain_restart_add(self, dominfo):
   1.142 +        self.restarts[dominfo.id] = dominfo
   1.143 +        log.info('Scheduling restart for domain: id=%s name=%s', dominfo.id, dominfo.name)
   1.144 +        self.domain_restarts_schedule()
   1.145              
   1.146      def domain_restart_cancel(self, id):
   1.147          """Cancel any restart scheduled for a domain.
   1.148 @@ -450,6 +450,7 @@ class XendDomain:
   1.149          """
   1.150          dominfo = self.restarts.get(id)
   1.151          if dominfo:
   1.152 +            log.info('Cancelling restart for domain: id=%s name=%s', dominfo.id, dominfo.name)
   1.153              dominfo.restart_cancel()
   1.154              del self.restarts[id]
   1.155  
   1.156 @@ -465,18 +466,18 @@ class XendDomain:
   1.157              # Remove it from the restarts.
   1.158              del self.restarts[id]
   1.159              try:
   1.160 -                log.info('domain_restarts> restart: id=%s config=%s', id, str(dominfo.config))
   1.161                  def cbok(dominfo):
   1.162 -                    log.info('Restarted domain %s as %s', id, dominfo.id)
   1.163 +                    log.info('Restarted domain id=%s as %s', id, dominfo.id)
   1.164                      self.domain_unpause(dominfo.id)
   1.165                  def cberr(err):
   1.166 -                    log.exception("Delayed exception restarting domain")
   1.167 +                    log.exception("Delayed exception restarting domain: id=%s", id)
   1.168                  deferred = self.domain_restart(dominfo)
   1.169                  deferred.addCallback(cbok)
   1.170                  deferred.addErrback(cberr)
   1.171              except:
   1.172 -                log.exception("Exception restarting domain")
   1.173 +                log.exception("Exception restarting domain: id=%s", id)
   1.174          if len(self.restarts):
   1.175 +            # Run again later if any restarts remain.
   1.176              self.refresh_schedule(delay=5)
   1.177          
   1.178      def final_domain_destroy(self, id):
   1.179 @@ -487,7 +488,7 @@ class XendDomain:
   1.180          dom = int(id)
   1.181          if dom <= 0:
   1.182              return 0
   1.183 -        log.info('Destroying domain %s', str(id))
   1.184 +        log.info('Destroying domain: id=%s', str(id))
   1.185          eserver.inject('xend.domain.destroy', id)
   1.186          dominfo = self.domain.get(id)
   1.187          if dominfo:
     2.1 --- a/tools/python/xen/xend/XendDomainInfo.py	Fri Jul 23 17:16:11 2004 +0000
     2.2 +++ b/tools/python/xen/xend/XendDomainInfo.py	Mon Jul 26 09:32:17 2004 +0000
     2.3 @@ -65,6 +65,9 @@ restart_modes = [
     2.4  STATE_RESTART_PENDING = 'pending'
     2.5  STATE_RESTART_BOOTING = 'booting'
     2.6  
     2.7 +STATE_VM_OK         = "ok"
     2.8 +STATE_VM_TERMINATED = "terminated"
     2.9 +
    2.10  def shutdown_reason(code):
    2.11      """Get a shutdown reason from a code.
    2.12  
    2.13 @@ -273,7 +276,11 @@ def vm_recreate(savedinfo, info):
    2.14      vm.memory = info['mem_kb']/1024
    2.15      start_time = sxp.child_value(savedinfo, 'start_time')
    2.16      if start_time is not None:
    2.17 -        vm.startTime = float(start_time)
    2.18 +        vm.start_time = float(start_time)
    2.19 +    vm.restart_state = sxp.child_value(savedinfo, 'restart_state')
    2.20 +    restart_time = sxp.child_value(savedinfo, 'restart_time')
    2.21 +    if restart_time is not None:
    2.22 +        vm.restart_time = float(restart_time)
    2.23      config = sxp.child_value(savedinfo, 'config')
    2.24      if config:
    2.25          d = vm.construct(config)
    2.26 @@ -339,15 +346,16 @@ def _vm_configure2(val, vm):
    2.27  class XendDomainInfo:
    2.28      """Virtual machine object."""
    2.29  
    2.30 -    STATE_OK = "ok"
    2.31 -    STATE_TERMINATED = "terminated"
    2.32 +    """Minimum time between domain restarts in seconds.
    2.33 +    """
    2.34 +    MINIMUM_RESTART_TIME = 10
    2.35  
    2.36      def __init__(self):
    2.37          self.recreate = 0
    2.38          self.config = None
    2.39          self.id = None
    2.40          self.dom = None
    2.41 -        self.startTime = None
    2.42 +        self.start_time = None
    2.43          self.name = None
    2.44          self.memory = None
    2.45          self.image = None
    2.46 @@ -361,11 +369,12 @@ class XendDomainInfo:
    2.47          self.blkif_backend = 0
    2.48          self.netif_backend = 0
    2.49          #todo: state: running, suspended
    2.50 -        self.state = self.STATE_OK
    2.51 +        self.state = STATE_VM_OK
    2.52          #todo: set to migrate info if migrating
    2.53          self.migrate = None
    2.54          self.restart_mode = RESTART_ONREBOOT
    2.55          self.restart_state = None
    2.56 +        self.restart_time = None
    2.57          self.console_port = None
    2.58  
    2.59      def setdom(self, dom):
    2.60 @@ -412,13 +421,17 @@ class XendDomainInfo:
    2.61              sxpr.append(['cpu', self.info['cpu']])
    2.62              sxpr.append(['cpu_time', self.info['cpu_time']/1e9])    
    2.63              
    2.64 -        if self.startTime:
    2.65 -            upTime =  time.time() - self.startTime  
    2.66 -            sxpr.append(['up_time', str(upTime) ])
    2.67 -            sxpr.append(['start_time', str(self.startTime) ])
    2.68 +        if self.start_time:
    2.69 +            up_time =  time.time() - self.start_time  
    2.70 +            sxpr.append(['up_time', str(up_time) ])
    2.71 +            sxpr.append(['start_time', str(self.start_time) ])
    2.72  
    2.73          if self.console:
    2.74              sxpr.append(self.console.sxpr())
    2.75 +        if self.restart_state:
    2.76 +            sxpr.append(['restart_state', self.restart_state])
    2.77 +        if self.restart_time:
    2.78 +            sxpr.append(['restart_time', str(self.restart_time)])
    2.79          if self.config:
    2.80              sxpr.append(['config', self.config])
    2.81          return sxpr
    2.82 @@ -551,13 +564,13 @@ class XendDomainInfo:
    2.83      def cleanup(self):
    2.84          """Cleanup vm resources: release devices.
    2.85          """
    2.86 -        self.state = self.STATE_TERMINATED
    2.87 +        self.state = STATE_VM_TERMINATED
    2.88          self.release_devices()
    2.89  
    2.90      def is_terminated(self):
    2.91          """Check if a domain has been terminated.
    2.92          """
    2.93 -        return self.state == self.STATE_TERMINATED
    2.94 +        return self.state == STATE_VM_TERMINATED
    2.95  
    2.96      def release_devices(self):
    2.97          """Release all vm devices.
    2.98 @@ -617,8 +630,8 @@ class XendDomainInfo:
    2.99          log.debug('init_domain> Created domain=%d name=%s memory=%d', dom, name, memory)
   2.100          self.setdom(dom)
   2.101  
   2.102 -        if self.startTime is None:
   2.103 -            self.startTime = time.time()
   2.104 +        if self.start_time is None:
   2.105 +            self.start_time = time.time()
   2.106  
   2.107      def build_domain(self, ostype, kernel, ramdisk, cmdline, vifs_n):
   2.108          """Build the domain boot image.
   2.109 @@ -628,7 +641,6 @@ class XendDomainInfo:
   2.110              log.warning('kernel cmdline too long, domain %d', self.dom)
   2.111          dom = self.dom
   2.112          buildfn = getattr(xc, '%s_build' % ostype)
   2.113 -        #print 'build_domain>', ostype, dom, kernel, cmdline, ramdisk
   2.114          flags = 0
   2.115          if self.netif_backend: flags |= SIF_NET_BE_DOMAIN
   2.116          if self.blkif_backend: flags |= SIF_BLK_BE_DOMAIN
   2.117 @@ -753,8 +765,31 @@ class XendDomainInfo:
   2.118      def restarting(self):
   2.119          self.restart_state = STATE_RESTART_PENDING
   2.120  
   2.121 +    def restart_pending(self):
   2.122 +        return self.restart_state == STATE_RESTART_PENDING
   2.123 +
   2.124 +    def restart_check(self):
   2.125 +        """Check if domain restart is OK.
   2.126 +        To prevent restart loops, raise an error it is
   2.127 +        less than MINIMUM_RESTART_TIME seconds since the last restart.
   2.128 +        """
   2.129 +        tnow = time.time()
   2.130 +        if self.restart_time is not None:
   2.131 +            tdelta = tnow - self.restart_time
   2.132 +            if tdelta < self.MINIMUM_RESTART_TIME:
   2.133 +                msg = 'VM %d restarting too fast' % self.dom
   2.134 +                log.error(msg)
   2.135 +                raise VmError(msg)
   2.136 +        self.restart_time = tnow
   2.137 +
   2.138      def restart(self):
   2.139 +        """Restart the domain after it has exited.
   2.140 +        Reuses the domain id and console port.
   2.141 +
   2.142 +        @return: deferred
   2.143 +        """
   2.144          try:
   2.145 +            self.restart_check()
   2.146              self.restart_state = STATE_RESTART_BOOTING
   2.147              d = self.construct(self.config)
   2.148          finally: