ia64/xen-unstable

changeset 12964:f911e945d959

[XEN][POWERPC] SMP/IPI/MB combined
This patch rolls up and rebases the following patches for submission
against current tip of tree:
* Memory barrier after SP store
* IPI support
* SMP support
The only changes from the previous submission other than trivial
fast-forward merges are to remove the ERAT flush, since it was comitted
seperately, and to make the status messages about waiting for remote
function completion ACKs only kick in after a full second has passed.
Note that this path REQUIRES that some form of the dom0 Linux patch
titled "Make Linux bail out of IPI vector reset" be applied.
Signed-off-by: Amos Waterland <apw@us.ibm.com>
Signed-off-by: Jimi Xenidis <jimix@watson.ibm.com>
Signed-off-by: Hollis Blanchard <hollisb@us.ibm.com>
author Jimi Xenidis <jimix@watson.ibm.com>
date Wed Nov 22 14:53:13 2006 -0500 (2006-11-22)
parents 5909f57c8c39
children 38b437a708a2
files xen/arch/powerpc/external.c xen/arch/powerpc/mpic.c xen/arch/powerpc/mpic_init.c xen/arch/powerpc/setup.c xen/arch/powerpc/smp.c xen/include/asm-powerpc/mach-default/irq_vectors.h xen/include/asm-powerpc/smp.h
line diff
     1.1 --- a/xen/arch/powerpc/external.c	Wed Nov 22 14:35:09 2006 -0500
     1.2 +++ b/xen/arch/powerpc/external.c	Wed Nov 22 14:53:13 2006 -0500
     1.3 @@ -82,7 +82,16 @@ void do_external(struct cpu_user_regs *r
     1.4  
     1.5      vec = xen_mpic_get_irq(regs);
     1.6  
     1.7 -    if (vec != -1) {
     1.8 +    if (vector_is_ipi(vec)) {
     1.9 +	/* do_IRQ is fundamentally broken for reliable IPI delivery.  */
    1.10 +	irq_desc_t *desc = &irq_desc[vec];
    1.11 +	regs->entry_vector = vec;
    1.12 +	spin_lock(&desc->lock);
    1.13 +	desc->handler->ack(vec);
    1.14 +	desc->action->handler(vector_to_irq(vec), desc->action->dev_id, regs);
    1.15 +	desc->handler->end(vec);
    1.16 +	spin_unlock(&desc->lock);
    1.17 +    } else if (vec != -1) {
    1.18          DBG("EE:0x%lx isrc: %d\n", regs->msr, vec);
    1.19          regs->entry_vector = vec;
    1.20          do_IRQ(regs);
    1.21 @@ -253,3 +262,24 @@ int ioapic_guest_write(unsigned long phy
    1.22      BUG_ON(val != val);
    1.23      return 0;
    1.24  }
    1.25 +
    1.26 +void send_IPI_mask(cpumask_t mask, int vector)
    1.27 +{
    1.28 +    unsigned int cpus;
    1.29 +    int const bits = 8 * sizeof(cpus);
    1.30 +
    1.31 +    switch(vector) {
    1.32 +    case CALL_FUNCTION_VECTOR:
    1.33 +    case EVENT_CHECK_VECTOR:
    1.34 +	break;
    1.35 +    default:
    1.36 +	BUG();
    1.37 +	return;
    1.38 +    }
    1.39 +
    1.40 +    BUG_ON(NR_CPUS > bits);
    1.41 +    BUG_ON(fls(mask.bits[0]) > bits);
    1.42 +
    1.43 +    cpus = mask.bits[0];
    1.44 +    mpic_send_ipi(vector, cpus);
    1.45 +}
     2.1 --- a/xen/arch/powerpc/mpic.c	Wed Nov 22 14:35:09 2006 -0500
     2.2 +++ b/xen/arch/powerpc/mpic.c	Wed Nov 22 14:53:13 2006 -0500
     2.3 @@ -27,10 +27,6 @@
     2.4  
     2.5  
     2.6  #define alloc_bootmem(x) xmalloc_bytes(x)
     2.7 -#define request_irq(irq, handler, f, devname, dev_id) \
     2.8 -    panic("IPI requested: %d: %p: %s: %p\n", irq, handler, devname, dev_id)
     2.9 -
    2.10 -typedef int irqreturn_t;
    2.11  
    2.12  #define IRQ_NONE	(0)
    2.13  #define IRQ_HANDLED	(1)
    2.14 @@ -97,11 +93,6 @@ typedef int irqreturn_t;
    2.15  #include <asm/mpic.h>
    2.16  #include <asm/smp.h>
    2.17  
    2.18 -static inline void smp_message_recv(int msg, struct pt_regs *regs)
    2.19 -{
    2.20 -    return;
    2.21 -}
    2.22 -
    2.23  #ifdef DEBUG
    2.24  #define DBG(fmt...) printk(fmt)
    2.25  #else
     3.1 --- a/xen/arch/powerpc/mpic_init.c	Wed Nov 22 14:35:09 2006 -0500
     3.2 +++ b/xen/arch/powerpc/mpic_init.c	Wed Nov 22 14:53:13 2006 -0500
     3.3 @@ -22,6 +22,7 @@
     3.4  #include <xen/init.h>
     3.5  #include <xen/lib.h>
     3.6  #include <asm/mpic.h>
     3.7 +#include <errno.h>
     3.8  #include "mpic_init.h"
     3.9  #include "oftree.h"
    3.10  #include "of-devtree.h"
    3.11 @@ -358,6 +359,42 @@ static struct hw_interrupt_type *share_m
    3.12  
    3.13  #endif
    3.14  
    3.15 +static unsigned int mpic_startup_ipi(unsigned int irq)
    3.16 +{
    3.17 +    mpic->hc_ipi.enable(irq);
    3.18 +    return 0;
    3.19 +}
    3.20 +
    3.21 +int request_irq(unsigned int irq,
    3.22 +		irqreturn_t (*handler)(int, void *, struct cpu_user_regs *),
    3.23 +		unsigned long irqflags, const char * devname, void *dev_id)
    3.24 +{
    3.25 +    int retval;
    3.26 +    struct irqaction *action;
    3.27 +    void (*func)(int, void *, struct cpu_user_regs *);
    3.28 +
    3.29 +    action = xmalloc(struct irqaction);
    3.30 +    if (!action) {
    3.31 +	BUG();
    3.32 +	return -ENOMEM;
    3.33 +    }
    3.34 +
    3.35 +    /* Xen's handler prototype is slightly different than Linux's.  */
    3.36 +    func = (void (*)(int, void *, struct cpu_user_regs *))handler;
    3.37 +
    3.38 +    action->handler = func;
    3.39 +    action->name = devname;
    3.40 +    action->dev_id = dev_id;
    3.41 +
    3.42 +    retval = setup_irq(irq, action);
    3.43 +    if (retval) {
    3.44 +	BUG();
    3.45 +	xfree(action);
    3.46 +    }
    3.47 +
    3.48 +    return retval;
    3.49 +}
    3.50 +
    3.51  struct hw_interrupt_type *xen_mpic_init(struct hw_interrupt_type *xen_irq)
    3.52  {
    3.53      unsigned int isu_size;
    3.54 @@ -397,6 +434,11 @@ struct hw_interrupt_type *xen_mpic_init(
    3.55      hit = share_mpic(&mpic->hc_irq, xen_irq);
    3.56  
    3.57      printk("%s: success\n", __func__);
    3.58 +
    3.59 +    mpic->hc_ipi.ack = xen_irq->ack;
    3.60 +    mpic->hc_ipi.startup = mpic_startup_ipi;
    3.61 +    mpic_request_ipis();
    3.62 +
    3.63      return hit;
    3.64  }
    3.65  
    3.66 @@ -406,3 +448,9 @@ int xen_mpic_get_irq(struct cpu_user_reg
    3.67  
    3.68  	return mpic_get_one_irq(mpic, regs);
    3.69  }
    3.70 +
    3.71 +int vector_is_ipi(int vector)
    3.72 +{
    3.73 +    BUG_ON(!mpic);
    3.74 +    return (mpic->ipi_offset <= vector) && (vector < mpic->ipi_offset + 4);
    3.75 +}
     4.1 --- a/xen/arch/powerpc/setup.c	Wed Nov 22 14:35:09 2006 -0500
     4.2 +++ b/xen/arch/powerpc/setup.c	Wed Nov 22 14:53:13 2006 -0500
     4.3 @@ -37,6 +37,7 @@
     4.4  #include <xen/keyhandler.h>
     4.5  #include <acm/acm_hooks.h>
     4.6  #include <public/version.h>
     4.7 +#include <asm/mpic.h>
     4.8  #include <asm/processor.h>
     4.9  #include <asm/desc.h>
    4.10  #include <asm/cache.h>
    4.11 @@ -88,6 +89,8 @@ struct ns16550_defaults ns16550;
    4.12  
    4.13  extern char __per_cpu_start[], __per_cpu_data_end[], __per_cpu_end[];
    4.14  
    4.15 +static struct domain *idle_domain;
    4.16 +
    4.17  volatile struct processor_area * volatile global_cpu_table[NR_CPUS];
    4.18  
    4.19  int is_kernel_text(unsigned long addr)
    4.20 @@ -159,8 +162,6 @@ static void percpu_free_unused_areas(voi
    4.21  
    4.22  static void __init start_of_day(void)
    4.23  {
    4.24 -    struct domain *idle_domain;
    4.25 -
    4.26      init_IRQ();
    4.27  
    4.28      scheduler_init();
    4.29 @@ -175,23 +176,6 @@ static void __init start_of_day(void)
    4.30      /* for some reason we need to set our own bit in the thread map */
    4.31      cpu_set(0, cpu_sibling_map[0]);
    4.32  
    4.33 -    percpu_free_unused_areas();
    4.34 -
    4.35 -    {
    4.36 -        /* FIXME: Xen assumes that an online CPU is a schedualable
    4.37 -         * CPU, but we just are not there yet. Remove this fragment when
    4.38 -         * scheduling processors actually works. */
    4.39 -        int cpuid;
    4.40 -
    4.41 -        printk("WARNING!: Taking all secondary CPUs offline\n");
    4.42 -
    4.43 -        for_each_online_cpu(cpuid) {
    4.44 -            if (cpuid == 0)
    4.45 -                continue;
    4.46 -            cpu_clear(cpuid, cpu_online_map);
    4.47 -        }
    4.48 -    }
    4.49 -
    4.50      initialize_keytable();
    4.51      /* Register another key that will allow for the the Harware Probe
    4.52       * to be contacted, this works with RiscWatch probes and should
    4.53 @@ -201,7 +185,6 @@ static void __init start_of_day(void)
    4.54      timer_init();
    4.55      serial_init_postirq();
    4.56      do_initcalls();
    4.57 -    schedulers_start();
    4.58  }
    4.59  
    4.60  void startup_cpu_idle_loop(void)
    4.61 @@ -234,6 +217,7 @@ static void init_parea(int cpuid)
    4.62      pa->whoami = cpuid;
    4.63      pa->hard_id = cpu_hard_id[cpuid];
    4.64      pa->hyp_stack_base = (void *)((ulong)stack + STACK_SIZE);
    4.65 +    mb();
    4.66  
    4.67      /* This store has the effect of invoking secondary_cpu_init.  */
    4.68      global_cpu_table[cpuid] = pa;
    4.69 @@ -263,9 +247,22 @@ static int kick_secondary_cpus(int maxcp
    4.70  /* This is the first C code that secondary processors invoke.  */
    4.71  int secondary_cpu_init(int cpuid, unsigned long r4)
    4.72  {
    4.73 +    struct vcpu *vcpu;
    4.74 +
    4.75      cpu_initialize(cpuid);
    4.76      smp_generic_take_timebase();
    4.77 +
    4.78 +    /* If we are online, we must be able to ACK IPIs.  */
    4.79 +    mpic_setup_this_cpu();
    4.80      cpu_set(cpuid, cpu_online_map);
    4.81 +
    4.82 +    vcpu = alloc_vcpu(idle_domain, cpuid, cpuid);
    4.83 +    BUG_ON(vcpu == NULL);
    4.84 +
    4.85 +    set_current(idle_domain->vcpu[cpuid]);
    4.86 +    idle_vcpu[cpuid] = current;
    4.87 +    startup_cpu_idle_loop();
    4.88 +
    4.89      while(1);
    4.90  }
    4.91  
    4.92 @@ -336,6 +333,10 @@ static void __init __start_xen(multiboot
    4.93          debugger_trap_immediate();
    4.94  #endif
    4.95  
    4.96 +    start_of_day();
    4.97 +
    4.98 +    mpic_setup_this_cpu();
    4.99 +
   4.100      /* Deal with secondary processors.  */
   4.101      if (opt_nosmp || ofd_boot_cpu == -1) {
   4.102          printk("nosmp: leaving secondary processors spinning forever\n");
   4.103 @@ -344,7 +345,11 @@ static void __init __start_xen(multiboot
   4.104          kick_secondary_cpus(max_cpus);
   4.105      }
   4.106  
   4.107 -    start_of_day();
   4.108 +    /* Secondary processors must be online before we call this.  */
   4.109 +    schedulers_start();
   4.110 +
   4.111 +    /* This cannot be called before secondary cpus are marked online.  */
   4.112 +    percpu_free_unused_areas();
   4.113  
   4.114      /* Create initial domain 0. */
   4.115      dom0 = domain_create(0, 0);
   4.116 @@ -403,6 +408,9 @@ static void __init __start_xen(multiboot
   4.117      console_end_sync();
   4.118  
   4.119      domain_unpause_by_systemcontroller(dom0);
   4.120 +#ifdef DEBUG_IPI
   4.121 +    ipi_torture_test();
   4.122 +#endif
   4.123      startup_cpu_idle_loop();
   4.124  }
   4.125  
     5.1 --- a/xen/arch/powerpc/smp.c	Wed Nov 22 14:35:09 2006 -0500
     5.2 +++ b/xen/arch/powerpc/smp.c	Wed Nov 22 14:53:13 2006 -0500
     5.3 @@ -22,6 +22,8 @@
     5.4  #include <xen/smp.h>
     5.5  #include <asm/flushtlb.h>
     5.6  #include <asm/debugger.h>
     5.7 +#include <asm/mpic.h>
     5.8 +#include <asm/mach-default/irq_vectors.h>
     5.9  
    5.10  int smp_num_siblings = 1;
    5.11  int smp_num_cpus = 1;
    5.12 @@ -50,7 +52,7 @@ void smp_send_event_check_mask(cpumask_t
    5.13  {
    5.14      cpu_clear(smp_processor_id(), mask);
    5.15      if (!cpus_empty(mask))
    5.16 -        unimplemented();
    5.17 +        send_IPI_mask(mask, EVENT_CHECK_VECTOR);
    5.18  }
    5.19  
    5.20  
    5.21 @@ -65,9 +67,21 @@ int smp_call_function(void (*func) (void
    5.22  
    5.23  void smp_send_stop(void)
    5.24  {
    5.25 -    unimplemented();
    5.26 +    BUG();
    5.27  }
    5.28  
    5.29 +struct call_data_struct {
    5.30 +    void (*func) (void *info);
    5.31 +    void *info;
    5.32 +    int wait;
    5.33 +    atomic_t started;
    5.34 +    atomic_t finished;
    5.35 +    cpumask_t selected;
    5.36 +};
    5.37 +
    5.38 +static DEFINE_SPINLOCK(call_lock);
    5.39 +static struct call_data_struct call_data;
    5.40 +
    5.41  int on_selected_cpus(
    5.42      cpumask_t selected,
    5.43      void (*func) (void *info),
    5.44 @@ -75,6 +89,115 @@ int on_selected_cpus(
    5.45      int retry,
    5.46      int wait)
    5.47  {
    5.48 -    unimplemented();
    5.49 -    return 0;
    5.50 +    int t, retval = 0, nr_cpus = cpus_weight(selected);
    5.51 +
    5.52 +    spin_lock(&call_lock);
    5.53 +
    5.54 +    call_data.func = func;
    5.55 +    call_data.info = info;
    5.56 +    call_data.wait = wait;
    5.57 +    call_data.wait = 1;  /* Until we get RCU around call_data.  */
    5.58 +    atomic_set(&call_data.started, 0);
    5.59 +    atomic_set(&call_data.finished, 0);
    5.60 +    mb();
    5.61 +
    5.62 +    send_IPI_mask(selected, CALL_FUNCTION_VECTOR);
    5.63 +
    5.64 +    /* We always wait for an initiation ACK from remote CPU.  */
    5.65 +    for (t = 0; atomic_read(&call_data.started) != nr_cpus; t++) {
    5.66 +	if (t && t % timebase_freq == 0) {
    5.67 +	    printk("IPI start stall: %d ACKS to %d SYNS\n", 
    5.68 +		   atomic_read(&call_data.started), nr_cpus);
    5.69 +	}
    5.70 +    }
    5.71 +
    5.72 +    /* If told to, we wait for a completion ACK from remote CPU.  */
    5.73 +    if (wait) {
    5.74 +	for (t = 0; atomic_read(&call_data.finished) != nr_cpus; t++) {
    5.75 +	    if (t > timebase_freq && t % timebase_freq == 0) {
    5.76 +		printk("IPI finish stall: %d ACKS to %d SYNS\n", 
    5.77 +		       atomic_read(&call_data.finished), nr_cpus);
    5.78 +	    }
    5.79 +	}
    5.80 +    }
    5.81 +
    5.82 +    spin_unlock(&call_lock);
    5.83 +
    5.84 +    return retval;
    5.85 +}
    5.86 +
    5.87 +void smp_call_function_interrupt(struct cpu_user_regs *regs)
    5.88 +{
    5.89 +
    5.90 +    void (*func)(void *info) = call_data.func;
    5.91 +    void *info = call_data.info;
    5.92 +    int wait = call_data.wait;
    5.93 +
    5.94 +    atomic_inc(&call_data.started);
    5.95 +    mb();
    5.96 +    (*func)(info);
    5.97 +    mb();
    5.98 +
    5.99 +    if (wait)
   5.100 +	atomic_inc(&call_data.finished);
   5.101 +
   5.102 +    return;
   5.103  }
   5.104 +
   5.105 +void smp_event_check_interrupt(void)
   5.106 +{
   5.107 +    /* We are knocked out of NAP state at least.  */
   5.108 +    return;
   5.109 +}
   5.110 +
   5.111 +void smp_message_recv(int msg, struct cpu_user_regs *regs)
   5.112 +{
   5.113 +    switch(msg) {
   5.114 +    case CALL_FUNCTION_VECTOR:
   5.115 +	smp_call_function_interrupt(regs);
   5.116 +	break;
   5.117 +    case EVENT_CHECK_VECTOR:
   5.118 +        smp_event_check_interrupt();
   5.119 +	break;
   5.120 +    default:
   5.121 +	BUG();
   5.122 +	break;
   5.123 +    }
   5.124 +}
   5.125 +
   5.126 +#ifdef DEBUG_IPI
   5.127 +static void debug_ipi_ack(void *info)
   5.128 +{
   5.129 +    return;
   5.130 +}
   5.131 +
   5.132 +void ipi_torture_test(void)
   5.133 +{
   5.134 +    int cpu;
   5.135 +    unsigned long before, after, delta;
   5.136 +    unsigned long min = ~0, max = 0, mean = 0, sum = 0, tick = 0;
   5.137 +    cpumask_t mask;
   5.138 +
   5.139 +    cpus_clear(mask);
   5.140 +
   5.141 +    while (tick < 1000000) {
   5.142 +	for_each_online_cpu(cpu) {
   5.143 +	    cpu_set(cpu, mask);
   5.144 +	    before = mftb();
   5.145 +	    on_selected_cpus(mask, debug_ipi_ack, NULL, 1, 1);
   5.146 +	    after = mftb();
   5.147 +	    cpus_clear(mask);
   5.148 +
   5.149 +	    delta = after - before;
   5.150 +	    if (delta > max) max = delta;
   5.151 +	    if (delta < min) min = delta;
   5.152 +	    sum += delta;
   5.153 +	    tick++;
   5.154 +	}
   5.155 +    }
   5.156 +
   5.157 +    mean = sum / tick;
   5.158 +
   5.159 +    printk("IPI tb ticks: min = %ld max = %ld mean = %ld\n", min, max, mean);
   5.160 +}
   5.161 +#endif
     6.1 --- a/xen/include/asm-powerpc/mach-default/irq_vectors.h	Wed Nov 22 14:35:09 2006 -0500
     6.2 +++ b/xen/include/asm-powerpc/mach-default/irq_vectors.h	Wed Nov 22 14:53:13 2006 -0500
     6.3 @@ -37,26 +37,10 @@
     6.4  #define FAST_TRAP -1 /* 0x80 */
     6.5  #define FIRST_SYSTEM_VECTOR	-1
     6.6  
     6.7 -#if 0
     6.8 -
     6.9 -/*
    6.10 - * Vectors 0-16 in some cases are used for ISA interrupts.
    6.11 - */
    6.12 +#define CALL_FUNCTION_VECTOR	0x0
    6.13 +#define EVENT_CHECK_VECTOR	0x1
    6.14  
    6.15 -/*
    6.16 - * Special IRQ vectors used by the SMP architecture, 0xf0-0xff
    6.17 - *
    6.18 - *  some of the following vectors are 'rare', they are merged
    6.19 - *  into a single vector (CALL_FUNCTION_VECTOR) to save vector space.
    6.20 - *  TLB, reschedule and local APIC vectors are performance-critical.
    6.21 - *
    6.22 - *  Vectors 0xf0-0xfa are free (reserved for future Linux use).
    6.23 - */
    6.24 -#define SPURIOUS_APIC_VECTOR	0xff
    6.25 -#define ERROR_APIC_VECTOR	0xfe
    6.26 -#define INVALIDATE_TLB_VECTOR	0xfd
    6.27 -#define EVENT_CHECK_VECTOR	0xfc
    6.28 -#define CALL_FUNCTION_VECTOR	0xfb
    6.29 +#if 0
    6.30  
    6.31  #define THERMAL_APIC_VECTOR	0xf0
    6.32  /*
     7.1 --- a/xen/include/asm-powerpc/smp.h	Wed Nov 22 14:35:09 2006 -0500
     7.2 +++ b/xen/include/asm-powerpc/smp.h	Wed Nov 22 14:53:13 2006 -0500
     7.3 @@ -35,4 +35,21 @@ extern cpumask_t cpu_sibling_map[];
     7.4  extern cpumask_t cpu_core_map[];
     7.5  extern void __devinit smp_generic_take_timebase(void);
     7.6  extern void __devinit smp_generic_give_timebase(void);
     7.7 +
     7.8 +#define SA_INTERRUPT	0x20000000u
     7.9 +typedef int irqreturn_t;
    7.10 +extern int request_irq(unsigned int irq,
    7.11 +    irqreturn_t (*handler)(int, void *, struct cpu_user_regs *),
    7.12 +    unsigned long irqflags, const char * devname, void *dev_id);
    7.13 +void smp_message_recv(int msg, struct cpu_user_regs *regs);
    7.14 +void smp_call_function_interrupt(struct cpu_user_regs *regs);
    7.15 +void smp_event_check_interrupt(void);
    7.16 +void send_IPI_mask(cpumask_t mask, int vector);
    7.17 +int vector_is_ipi(int vector);
    7.18 +
    7.19 +#undef DEBUG_IPI
    7.20 +#ifdef DEBUG_IPI
    7.21 +void ipi_torture_test(void);
    7.22  #endif
    7.23 +
    7.24 +#endif