direct-io.hg

changeset 5691:88c2d410979f

I updated the vcpu_to_cpu string creation to include a field separator,
which gets rid of the -1 -> # hack and works for cpus > 9.

I ran into some issues with stale vcpu_to_cpu lists when running the
hotplug subprogram. I would take a vcpu offline, and then issue the
command to bring it back and the vcpu_to_cpu list would not have changed
to indicate the the vcpu actually went down. If I injected a xm list -v
(which always showed the correct mapping) then subsequent hotplug
commands would see the state change and fire off the hotplug request. I
don't know that not sending the event when not changing state saves that
much work so I took the state check out and now just send the hotplug
event directly.

> Also the whole hotplug stuff is still missing interrupt re-routing
> when a vcpu is taken down. To do this, we need an evtchn operation to
> change the vcpu affinity of a port by changing notify_vcpu_id.

I don't fully understand all of the mappings that are happening, so this
part of the patch might be way off. In any case, I've added a new
evtchn op to set the notify_vcpu_id field of a channel. I updated the
HOTPLUG_CPU code to use the new routines when bringing cpus up and down.
When taking down a cpu, I route the IPI irq channels to CPU 0, and when
the cpu comes up, it re-routes the channels back to the awakened CPU.

From: Ryan Harper <ryanh@us.ibm.com>
Signed-off-by: ian@xensource.com
author iap10@freefall.cl.cam.ac.uk
date Wed Jul 06 22:23:18 2005 +0000 (2005-07-06)
parents 32fb371cc283
children 2db810d7985f
files linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c tools/python/xen/xend/XendDomainInfo.py tools/python/xen/xm/main.py xen/common/event_channel.c xen/include/public/event_channel.h
line diff
     1.1 --- a/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c	Wed Jul 06 18:57:54 2005 +0000
     1.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/i386/kernel/smpboot.c	Wed Jul 06 22:23:18 2005 +0000
     1.3 @@ -103,6 +103,11 @@ static int trampoline_exec;
     1.4  DEFINE_PER_CPU(int, cpu_state) = { 0 };
     1.5  #endif
     1.6  
     1.7 +static DEFINE_PER_CPU(int, resched_irq);
     1.8 +static DEFINE_PER_CPU(int, callfunc_irq);
     1.9 +static char resched_name[NR_CPUS][15];
    1.10 +static char callfunc_name[NR_CPUS][15];
    1.11 +
    1.12  #if 0
    1.13  /*
    1.14   * Currently trivial. Write the real->protected mode
    1.15 @@ -1328,6 +1333,10 @@ static int __devinit cpu_enable(unsigned
    1.16  	while (!cpu_online(cpu))
    1.17  		cpu_relax();
    1.18  
    1.19 +   /* re-route bound IRQs 0 to cpu */
    1.20 +   rebind_evtchn_from_irq(0, cpu,  per_cpu(resched_irq, cpu));
    1.21 +   rebind_evtchn_from_irq(0, cpu, per_cpu(callfunc_irq, cpu));
    1.22 +
    1.23  	fixup_irqs(cpu_online_map);
    1.24  	/* counter the disable in fixup_irqs() */
    1.25  	local_irq_enable();
    1.26 @@ -1357,6 +1366,11 @@ int __cpu_disable(void)
    1.27  
    1.28  	cpu_clear(cpu, map);
    1.29  	fixup_irqs(map);
    1.30 +
    1.31 +   /* re-route IRQs from dead vcpu to another */
    1.32 +   rebind_evtchn_from_irq(cpu, 0,  per_cpu(resched_irq, cpu));
    1.33 +   rebind_evtchn_from_irq(cpu, 0, per_cpu(callfunc_irq, cpu));
    1.34 +
    1.35  	/* It's now safe to remove this processor from the online map */
    1.36  	cpu_clear(cpu, cpu_online_map);
    1.37  
    1.38 @@ -1514,11 +1528,6 @@ void __init smp_cpus_done(unsigned int m
    1.39  extern irqreturn_t smp_reschedule_interrupt(int, void *, struct pt_regs *);
    1.40  extern irqreturn_t smp_call_function_interrupt(int, void *, struct pt_regs *);
    1.41  
    1.42 -static DEFINE_PER_CPU(int, resched_irq);
    1.43 -static DEFINE_PER_CPU(int, callfunc_irq);
    1.44 -static char resched_name[NR_CPUS][15];
    1.45 -static char callfunc_name[NR_CPUS][15];
    1.46 -
    1.47  void __init smp_intr_init(void)
    1.48  {
    1.49  	int cpu = smp_processor_id();
     2.1 --- a/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c	Wed Jul 06 18:57:54 2005 +0000
     2.2 +++ b/linux-2.6.11-xen-sparse/arch/xen/kernel/evtchn.c	Wed Jul 06 22:23:18 2005 +0000
     2.3 @@ -271,6 +271,38 @@ int bind_ipi_on_cpu_to_irq(int cpu, int 
     2.4      return irq;
     2.5  }
     2.6  
     2.7 +void rebind_evtchn_from_ipi(int cpu, int newcpu, int ipi)
     2.8 +{
     2.9 +    evtchn_op_t op;
    2.10 +    int evtchn = per_cpu(ipi_to_evtchn, cpu)[ipi];
    2.11 +
    2.12 +    spin_lock(&irq_mapping_update_lock);
    2.13 +
    2.14 +    op.cmd          = EVTCHNOP_rebind;
    2.15 +    op.u.rebind.port = evtchn;
    2.16 +    op.u.rebind.vcpu = newcpu;
    2.17 +    if ( HYPERVISOR_event_channel_op(&op) != 0 )
    2.18 +       printk(KERN_INFO "Failed to rebind IPI%d to CPU%d\n",ipi,newcpu);
    2.19 +
    2.20 +    spin_unlock(&irq_mapping_update_lock);
    2.21 +}
    2.22 +
    2.23 +void rebind_evtchn_from_irq(int cpu, int newcpu, int irq)
    2.24 +{
    2.25 +    evtchn_op_t op;
    2.26 +    int evtchn = irq_to_evtchn[irq];
    2.27 +
    2.28 +    spin_lock(&irq_mapping_update_lock);
    2.29 +
    2.30 +    op.cmd          = EVTCHNOP_rebind;
    2.31 +    op.u.rebind.port = evtchn;
    2.32 +    op.u.rebind.vcpu = newcpu;
    2.33 +    if ( HYPERVISOR_event_channel_op(&op) != 0 )
    2.34 +       printk(KERN_INFO "Failed to rebind IRQ%d to CPU%d\n",irq,newcpu);
    2.35 +
    2.36 +    spin_unlock(&irq_mapping_update_lock);
    2.37 +}
    2.38 +
    2.39  void unbind_ipi_on_cpu_from_irq(int cpu, int ipi)
    2.40  {
    2.41      evtchn_op_t op;
     3.1 --- a/tools/python/xen/xend/XendDomainInfo.py	Wed Jul 06 18:57:54 2005 +0000
     3.2 +++ b/tools/python/xen/xend/XendDomainInfo.py	Wed Jul 06 22:23:18 2005 +0000
     3.3 @@ -423,8 +423,10 @@ class XendDomainInfo:
     3.4              sxpr.append(['cpu_time', self.info['cpu_time']/1e9])    
     3.5              sxpr.append(['vcpus', self.info['vcpus']])
     3.6              sxpr.append(['cpumap', self.info['cpumap']])
     3.7 -            sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x),
     3.8 -                        self.info['vcpu_to_cpu'][0:self.info['vcpus']]))])
     3.9 +            # build a string, using '|' to seperate items, show only up
    3.10 +            # to number of vcpus in domain, and trim the trailing '|'
    3.11 +            sxpr.append(['vcpu_to_cpu', ''.join(map(lambda x: str(x)+'|',
    3.12 +                        self.info['vcpu_to_cpu'][0:self.info['vcpus']]))[:-1]])
    3.13              
    3.14          if self.start_time:
    3.15              up_time =  time.time() - self.start_time  
     4.1 --- a/tools/python/xen/xm/main.py	Wed Jul 06 18:57:54 2005 +0000
     4.2 +++ b/tools/python/xen/xm/main.py	Wed Jul 06 22:23:18 2005 +0000
     4.3 @@ -410,8 +410,7 @@ class ProgList(Prog):
     4.4          print 'Name              Id  VCPU  CPU  CPUMAP'
     4.5          for dom in doms:
     4.6              info = server.xend_domain(dom)
     4.7 -            # XXX this is quite broken for cpu's > 9
     4.8 -            vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '?').replace('-1','#')
     4.9 +            vcpu_to_cpu = sxp.child_value(info, 'vcpu_to_cpu', '-1').split('|')
    4.10              cpumap = sxp.child_value(info, 'cpumap', [])
    4.11              mask = ((int(sxp.child_value(info, 'vcpus', '0')))**2) - 1
    4.12              count = 0
    4.13 @@ -420,10 +419,7 @@ class ProgList(Prog):
    4.14                  d['name']   = sxp.child_value(info, 'name', '??')
    4.15                  d['dom']    = int(sxp.child_value(info, 'id', '-1'))
    4.16                  d['vcpu']   = int(count)
    4.17 -                if cpu == "#":
    4.18 -                    d['cpu']    = int("-1")
    4.19 -                else:
    4.20 -                    d['cpu']    = int(cpu)
    4.21 +                d['cpu']    = int(cpu)
    4.22                  d['cpumap'] = int(cpumap[count])&mask
    4.23                  count = count + 1
    4.24                  print ("%(name)-16s %(dom)3d  %(vcpu)4d  %(cpu)3d  0x%(cpumap)x" % d)
    4.25 @@ -593,15 +589,7 @@ domain DOM"""
    4.26          state = int(args[3])
    4.27          dom = server.xend_domain(name)
    4.28          id = sxp.child_value(dom, 'id')
    4.29 -        vcpu_to_cpu = sxp.child_value(dom, 'vcpu_to_cpu', '-1')
    4.30 -        # only send state change if states differ 
    4.31 -        try:
    4.32 -            # (down going up) or (up going down)
    4.33 -            if (vcpu_to_cpu[vcpu] == "-1" and state == 1) or \
    4.34 -               (vcpu_to_cpu[vcpu] != "-1" and state == 0):
    4.35 -                server.xend_domain_vcpu_hotplug(id, vcpu, state)
    4.36 -        except IndexError:
    4.37 -            print "Invalid VCPU(%d)"%(vcpu)
    4.38 +        server.xend_domain_vcpu_hotplug(id, vcpu, state)
    4.39  
    4.40  xm.prog(ProgVcpuhotplug)
    4.41  
     5.1 --- a/xen/common/event_channel.c	Wed Jul 06 18:57:54 2005 +0000
     5.2 +++ b/xen/common/event_channel.c	Wed Jul 06 22:23:18 2005 +0000
     5.3 @@ -579,6 +579,29 @@ static long evtchn_status(evtchn_status_
     5.4      return rc;
     5.5  }
     5.6  
     5.7 +static long evtchn_rebind(evtchn_rebind_t *bind) 
     5.8 +{
     5.9 +    struct domain *d    = current->domain;
    5.10 +    int            port = bind->port;
    5.11 +    int            vcpu = bind->vcpu;
    5.12 +    struct evtchn *chn;
    5.13 +    long             rc = 0;
    5.14 +
    5.15 +    spin_lock(&d->evtchn_lock);
    5.16 +
    5.17 +    if ( !port_is_valid(d, port) )
    5.18 +    {
    5.19 +        rc = -EINVAL;
    5.20 +        goto out;
    5.21 +    }
    5.22 +
    5.23 +    chn = evtchn_from_port(d, port);
    5.24 +    chn->notify_vcpu_id = vcpu;
    5.25 +
    5.26 + out:
    5.27 +    spin_unlock(&d->evtchn_lock);
    5.28 +    return rc;
    5.29 +}
    5.30  
    5.31  long do_event_channel_op(evtchn_op_t *uop)
    5.32  {
    5.33 @@ -637,6 +660,12 @@ long do_event_channel_op(evtchn_op_t *uo
    5.34              rc = -EFAULT;
    5.35          break;
    5.36  
    5.37 +    case EVTCHNOP_rebind:
    5.38 +        rc = evtchn_rebind(&op.u.rebind);
    5.39 +        if ( (rc == 0) && (copy_to_user(uop, &op, sizeof(op)) != 0) )
    5.40 +            rc = -EFAULT;
    5.41 +        break;
    5.42 +
    5.43      default:
    5.44          rc = -ENOSYS;
    5.45          break;
     6.1 --- a/xen/include/public/event_channel.h	Wed Jul 06 18:57:54 2005 +0000
     6.2 +++ b/xen/include/public/event_channel.h	Wed Jul 06 22:23:18 2005 +0000
     6.3 @@ -158,6 +158,13 @@ typedef struct evtchn_status {
     6.4      } u;
     6.5  } evtchn_status_t;
     6.6  
     6.7 +#define EVTCHNOP_rebind        8
     6.8 +typedef struct {
     6.9 +    /* IN parameters. */
    6.10 +    u32 port;                         /*  0 */
    6.11 +    u32 vcpu;                         /*  4 */
    6.12 +} evtchn_rebind_t; /* 8 bytes */
    6.13 +
    6.14  typedef struct evtchn_op {
    6.15      u32 cmd; /* EVTCHNOP_* */
    6.16      union {
    6.17 @@ -169,6 +176,7 @@ typedef struct evtchn_op {
    6.18          evtchn_close_t            close;
    6.19          evtchn_send_t             send;
    6.20          evtchn_status_t           status;
    6.21 +        evtchn_rebind_t           rebind;
    6.22      } u;
    6.23  } evtchn_op_t;
    6.24