1.1 --- a/xen/arch/x86/io_apic.c	Sat Apr 15 19:25:21 2006 +0100
     1.2 +++ b/xen/arch/x86/io_apic.c	Sun Apr 16 15:04:21 2006 +0100
     1.3 @@ -202,18 +202,6 @@ static void __level_IO_APIC_irq (unsigne
     1.4      __modify_IO_APIC_irq(irq, 0x00008000, 0);
     1.5  }
     1.6  
     1.7 -/* mask = 1, trigger = 0 */
     1.8 -static void __mask_and_edge_IO_APIC_irq (unsigned int irq)
     1.9 -{
    1.10 -    __modify_IO_APIC_irq(irq, 0x00010000, 0x00008000);
    1.11 -}
    1.12 -
    1.13 -/* mask = 0, trigger = 1 */
    1.14 -static void __unmask_and_level_IO_APIC_irq (unsigned int irq)
    1.15 -{
    1.16 -    __modify_IO_APIC_irq(irq, 0x00008000, 0x00010000);
    1.17 -}
    1.18 -
    1.19  static void mask_IO_APIC_irq (unsigned int irq)
    1.20  {
    1.21      unsigned long flags;
    1.22 @@ -1395,7 +1383,8 @@ static void end_level_ioapic_irq (unsign
    1.23  
    1.24      if ( !ioapic_ack_new )
    1.25      {
    1.26 -        unmask_IO_APIC_irq(irq);
    1.27 +        if ( !(irq_desc[IO_APIC_VECTOR(irq)].status & IRQ_DISABLED) )
    1.28 +            unmask_IO_APIC_irq(irq);
    1.29          return;
    1.30      }
    1.31  
    1.32 @@ -1427,8 +1416,11 @@ static void end_level_ioapic_irq (unsign
    1.33      if (!(v & (1 << (i & 0x1f)))) {
    1.34          atomic_inc(&irq_mis_count);
    1.35          spin_lock(&ioapic_lock);
    1.36 -        __mask_and_edge_IO_APIC_irq(irq);
    1.37 -        __unmask_and_level_IO_APIC_irq(irq);
    1.38 +        __mask_IO_APIC_irq(irq);
    1.39 +        __edge_IO_APIC_irq(irq);
    1.40 +        __level_IO_APIC_irq(irq);
    1.41 +        if ( !(irq_desc[IO_APIC_VECTOR(irq)].status & IRQ_DISABLED) )
    1.42 +            __unmask_IO_APIC_irq(irq);
    1.43          spin_unlock(&ioapic_lock);
    1.44      }
    1.45  }

     2.1 --- a/xen/arch/x86/irq.c	Sat Apr 15 19:25:21 2006 +0100
     2.2 +++ b/xen/arch/x86/irq.c	Sun Apr 16 15:04:21 2006 +0100
     2.3 @@ -151,7 +151,7 @@ typedef struct {
     2.4      u8 ack_type;
     2.5  #define ACKTYPE_NONE   0     /* No final acknowledgement is required */
     2.6  #define ACKTYPE_UNMASK 1     /* Unmask PIC hardware (from any CPU)   */
     2.7 -#define ACKTYPE_LAPIC_EOI  2 /* EOI on the CPU that was interrupted  */
     2.8 +#define ACKTYPE_EOI    2     /* EOI on the CPU that was interrupted  */
     2.9      cpumask_t cpu_eoi_map;   /* CPUs that need to EOI this interrupt */
    2.10      struct domain *guest[IRQ_MAX_GUESTS];
    2.11  } irq_guest_action_t;
    2.12 @@ -161,10 +161,10 @@ typedef struct {
    2.13   * order, as only the current highest-priority pending irq can be EOIed.
    2.14   */
    2.15  static struct {
    2.16 -    u8 vector;
    2.17 -    u8 ready_to_end;
    2.18 -} pending_lapic_eoi[NR_CPUS][NR_VECTORS] __cacheline_aligned;
    2.19 -#define pending_lapic_eoi_sp(cpu) (pending_lapic_eoi[cpu][NR_VECTORS-1].vector)
    2.20 +    u8 vector; /* Vector awaiting EOI */
    2.21 +    u8 ready;  /* Ready for EOI now?  */
    2.22 +} pending_eoi[NR_CPUS][NR_VECTORS] __cacheline_aligned;
    2.23 +#define pending_eoi_sp(cpu) (pending_eoi[cpu][NR_VECTORS-1].vector)
    2.24  
    2.25  static void __do_IRQ_guest(int vector)
    2.26  {
    2.27 @@ -176,20 +176,21 @@ static void __do_IRQ_guest(int vector)
    2.28  
    2.29      if ( unlikely(action->nr_guests == 0) )
    2.30      {
    2.31 -        /* An interrupt may slip through while freeing a LAPIC_EOI irq. */
    2.32 -        ASSERT(action->ack_type == ACKTYPE_LAPIC_EOI);
    2.33 +        /* An interrupt may slip through while freeing an ACKTYPE_EOI irq. */
    2.34 +        ASSERT(action->ack_type == ACKTYPE_EOI);
    2.35 +        ASSERT(desc->status & IRQ_DISABLED);
    2.36          desc->handler->end(vector);
    2.37          return;
    2.38      }
    2.39  
    2.40 -    if ( action->ack_type == ACKTYPE_LAPIC_EOI )
    2.41 +    if ( action->ack_type == ACKTYPE_EOI )
    2.42      {
    2.43 -        sp = pending_lapic_eoi_sp(cpu);
    2.44 -        ASSERT((sp == 0) || (pending_lapic_eoi[cpu][sp-1].vector < vector));
    2.45 +        sp = pending_eoi_sp(cpu);
    2.46 +        ASSERT((sp == 0) || (pending_eoi[cpu][sp-1].vector < vector));
    2.47          ASSERT(sp < (NR_VECTORS-1));
    2.48 -        pending_lapic_eoi[cpu][sp].vector = vector;
    2.49 -        pending_lapic_eoi[cpu][sp].ready_to_end = 0;
    2.50 -        pending_lapic_eoi_sp(cpu) = sp+1;
    2.51 +        pending_eoi[cpu][sp].vector = vector;
    2.52 +        pending_eoi[cpu][sp].ready = 0;
    2.53 +        pending_eoi_sp(cpu) = sp+1;
    2.54          cpu_set(cpu, action->cpu_eoi_map);
    2.55      }
    2.56  
    2.57 @@ -203,47 +204,93 @@ static void __do_IRQ_guest(int vector)
    2.58      }
    2.59  }
    2.60  
    2.61 -static void end_guest_irq(void *data)
    2.62 +/* Flush all ready EOIs from the top of this CPU's pending-EOI stack. */
    2.63 +static void flush_ready_eoi(void *unused)
    2.64  {
    2.65 -    irq_desc_t         *desc = data;
    2.66 +    irq_desc_t *desc;
    2.67 +    int         vector, sp, cpu = smp_processor_id();
    2.68 +
    2.69 +    ASSERT(!local_irq_is_enabled());
    2.70 +
    2.71 +    sp = pending_eoi_sp(cpu);
    2.72 +
    2.73 +    while ( (--sp >= 0) && pending_eoi[cpu][sp].ready )
    2.74 +    {
    2.75 +        vector = pending_eoi[cpu][sp].vector;
    2.76 +        desc = &irq_desc[vector];
    2.77 +        spin_lock(&desc->lock);
    2.78 +        desc->handler->end(vector);
    2.79 +        spin_unlock(&desc->lock);
    2.80 +    }
    2.81 +
    2.82 +    pending_eoi_sp(cpu) = sp+1;
    2.83 +}
    2.84 +
    2.85 +static void __set_eoi_ready(irq_desc_t *desc)
    2.86 +{
    2.87      irq_guest_action_t *action = (irq_guest_action_t *)desc->action;
    2.88 -    unsigned long       flags;
    2.89      int                 vector, sp, cpu = smp_processor_id();
    2.90  
    2.91      vector = desc - irq_desc;
    2.92  
    2.93 -    spin_lock_irqsave(&desc->lock, flags);
    2.94 +    if ( !(desc->status & IRQ_GUEST) ||
    2.95 +         (action->in_flight != 0) ||
    2.96 +         !test_and_clear_bit(cpu, &action->cpu_eoi_map) )
    2.97 +        return;
    2.98  
    2.99 -    if ( (desc->status & IRQ_GUEST) &&
   2.100 -         (action->in_flight == 0) &&
   2.101 -         test_and_clear_bit(cpu, &action->cpu_eoi_map) )
   2.102 +    sp = pending_eoi_sp(cpu);
   2.103 +    do {
   2.104 +        ASSERT(sp > 0);
   2.105 +    } while ( pending_eoi[cpu][--sp].vector != vector );
   2.106 +    ASSERT(!pending_eoi[cpu][sp].ready);
   2.107 +    pending_eoi[cpu][sp].ready = 1;
   2.108 +}
   2.109 +
   2.110 +/* Mark specified IRQ as ready-for-EOI (if it really is) and attempt to EOI. */
   2.111 +static void set_eoi_ready(void *data)
   2.112 +{
   2.113 +    irq_desc_t *desc = data;
   2.114 +
   2.115 +    ASSERT(!local_irq_is_enabled());
   2.116 +
   2.117 +    spin_lock(&desc->lock);
   2.118 +    __set_eoi_ready(desc);
   2.119 +    spin_unlock(&desc->lock);
   2.120 +
   2.121 +    flush_ready_eoi(NULL);
   2.122 +}
   2.123 +
   2.124 +/*
   2.125 + * Forcibly flush all pending EOIs on this CPU by emulating end-of-ISR
   2.126 + * notifications from guests. The caller of this function must ensure that
   2.127 + * all CPUs execute flush_ready_eoi().
   2.128 + */
   2.129 +static void flush_all_pending_eoi(void *unused)
   2.130 +{
   2.131 +    irq_desc_t         *desc;
   2.132 +    irq_guest_action_t *action;
   2.133 +    int                 i, vector, sp, cpu = smp_processor_id();
   2.134 +
   2.135 +    ASSERT(!local_irq_is_enabled());
   2.136 +
   2.137 +    sp = pending_eoi_sp(cpu);
   2.138 +    while ( --sp >= 0 )
   2.139      {
   2.140 -        sp = pending_lapic_eoi_sp(cpu);
   2.141 -        do {
   2.142 -            ASSERT(sp > 0);
   2.143 -        } while ( pending_lapic_eoi[cpu][--sp].vector != vector );
   2.144 -        ASSERT(!pending_lapic_eoi[cpu][sp].ready_to_end);
   2.145 -        pending_lapic_eoi[cpu][sp].ready_to_end = 1;
   2.146 +        if ( pending_eoi[cpu][sp].ready )
   2.147 +            continue;
   2.148 +        vector = pending_eoi[cpu][sp].vector;
   2.149 +        desc = &irq_desc[vector];
   2.150 +        spin_lock(&desc->lock);
   2.151 +        action = (irq_guest_action_t *)desc->action;
   2.152 +        ASSERT(action->ack_type == ACKTYPE_EOI);
   2.153 +        ASSERT(desc->status & IRQ_GUEST);
   2.154 +        for ( i = 0; i < action->nr_guests; i++ )
   2.155 +            clear_bit(vector_to_irq(vector), &action->guest[i]->pirq_mask);
   2.156 +        action->in_flight = 0;
   2.157 +        spin_unlock(&desc->lock);
   2.158      }
   2.159  
   2.160 -    for ( ; ; )
   2.161 -    {
   2.162 -        sp = pending_lapic_eoi_sp(cpu);
   2.163 -        if ( (sp == 0) || !pending_lapic_eoi[cpu][sp-1].ready_to_end )
   2.164 -        {
   2.165 -            spin_unlock_irqrestore(&desc->lock, flags);    
   2.166 -            return;
   2.167 -        }
   2.168 -        if ( pending_lapic_eoi[cpu][sp-1].vector != vector )
   2.169 -        {
   2.170 -            spin_unlock(&desc->lock);
   2.171 -            vector = pending_lapic_eoi[cpu][sp-1].vector;
   2.172 -            desc = &irq_desc[vector];
   2.173 -            spin_lock(&desc->lock);
   2.174 -        }
   2.175 -        desc->handler->end(vector);
   2.176 -        pending_lapic_eoi_sp(cpu) = sp-1;
   2.177 -    }
   2.178 +    flush_ready_eoi(NULL);
   2.179  }
   2.180  
   2.181  int pirq_guest_unmask(struct domain *d)
   2.182 @@ -262,6 +309,7 @@ int pirq_guest_unmask(struct domain *d)
   2.183          action = (irq_guest_action_t *)desc->action;
   2.184  
   2.185          spin_lock_irq(&desc->lock);
   2.186 +
   2.187          if ( !test_bit(d->pirq_to_evtchn[pirq], &s->evtchn_mask[0]) &&
   2.188               test_and_clear_bit(pirq, &d->pirq_mask) )
   2.189          {
   2.190 @@ -273,14 +321,22 @@ int pirq_guest_unmask(struct domain *d)
   2.191                  cpu_eoi_map = action->cpu_eoi_map;
   2.192              }
   2.193          }
   2.194 -        spin_unlock_irq(&desc->lock);
   2.195  
   2.196          if ( __test_and_clear_bit(cpu, &cpu_eoi_map) )
   2.197 -            end_guest_irq(desc);
   2.198 +        {
   2.199 +            __set_eoi_ready(desc);
   2.200 +            spin_unlock(&desc->lock);
   2.201 +            flush_ready_eoi(NULL);
   2.202 +            local_irq_enable();
   2.203 +        }
   2.204 +        else
   2.205 +        {
   2.206 +            spin_unlock_irq(&desc->lock);
   2.207 +        }
   2.208  
   2.209          if ( !cpus_empty(cpu_eoi_map) )
   2.210          {
   2.211 -            on_selected_cpus(cpu_eoi_map, end_guest_irq, desc, 1, 0);
   2.212 +            on_selected_cpus(cpu_eoi_map, set_eoi_ready, desc, 1, 0);
   2.213              cpu_eoi_map = CPU_MASK_NONE;
   2.214          }
   2.215      }
   2.216 @@ -316,7 +372,7 @@ int pirq_acktype(int irq)
   2.217       * on which they were received. This is because we tickle the LAPIC to EOI.
   2.218       */
   2.219      if ( !strcmp(desc->handler->typename, "IO-APIC-level") )
   2.220 -        return ioapic_ack_new ? ACKTYPE_LAPIC_EOI : ACKTYPE_UNMASK;
   2.221 +        return ioapic_ack_new ? ACKTYPE_EOI : ACKTYPE_UNMASK;
   2.222  
   2.223      BUG();
   2.224      return 0;
   2.225 @@ -334,6 +390,7 @@ int pirq_guest_bind(struct vcpu *v, int 
   2.226      if ( (irq < 0) || (irq >= NR_IRQS) )
   2.227          return -EINVAL;
   2.228  
   2.229 + retry:
   2.230      vector = irq_to_vector(irq);
   2.231      if ( vector == 0 )
   2.232          return -EINVAL;
   2.233 @@ -385,6 +442,18 @@ int pirq_guest_bind(struct vcpu *v, int 
   2.234          rc = -EBUSY;
   2.235          goto out;
   2.236      }
   2.237 +    else if ( action->nr_guests == 0 )
   2.238 +    {
   2.239 +        /*
   2.240 +         * Indicates that an ACKTYPE_EOI interrupt is being released.
   2.241 +         * Wait for that to happen before continuing.
   2.242 +         */
   2.243 +        ASSERT(action->ack_type == ACKTYPE_EOI);
   2.244 +        ASSERT(desc->status & IRQ_DISABLED);
   2.245 +        spin_unlock_irqrestore(&desc->lock, flags);
   2.246 +        cpu_relax();
   2.247 +        goto retry;
   2.248 +    }
   2.249  
   2.250      if ( action->nr_guests == IRQ_MAX_GUESTS )
   2.251      {
   2.252 @@ -428,27 +497,16 @@ int pirq_guest_unbind(struct domain *d, 
   2.253               (--action->in_flight == 0) )
   2.254              desc->handler->end(vector);
   2.255          break;
   2.256 -    case ACKTYPE_LAPIC_EOI:
   2.257 -        if ( test_and_clear_bit(irq, &d->pirq_mask) )
   2.258 -            --action->in_flight;
   2.259 -        while ( action->in_flight == 0 )
   2.260 +    case ACKTYPE_EOI:
   2.261 +        /* NB. If #guests == 0 then we clear the eoi_map later on. */
   2.262 +        if ( test_and_clear_bit(irq, &d->pirq_mask) &&
   2.263 +             (--action->in_flight == 0) &&
   2.264 +             (action->nr_guests != 0) )
   2.265          {
   2.266 -            /* We cannot release guest info until all pending ACKs are done. */
   2.267              cpu_eoi_map = action->cpu_eoi_map;
   2.268 -            if ( cpus_empty(cpu_eoi_map) )
   2.269 -                break;
   2.270 -
   2.271 -            /* We cannot hold the lock while interrupting other CPUs. */
   2.272              spin_unlock_irqrestore(&desc->lock, flags);    
   2.273 -            on_selected_cpus(cpu_eoi_map, end_guest_irq, desc, 1, 1);
   2.274 +            on_selected_cpus(cpu_eoi_map, set_eoi_ready, desc, 1, 0);
   2.275              spin_lock_irqsave(&desc->lock, flags);
   2.276 -
   2.277 -            /* The world can change while we do not hold the lock. */
   2.278 -            if ( !(desc->status & IRQ_GUEST) )
   2.279 -                goto out;
   2.280 -            if ( (action->ack_type != ACKTYPE_LAPIC_EOI) ||
   2.281 -                 (action->nr_guests != 0) )
   2.282 -                break;
   2.283          }
   2.284          break;
   2.285      }
   2.286 @@ -459,12 +517,31 @@ int pirq_guest_unbind(struct domain *d, 
   2.287          goto out;
   2.288  
   2.289      BUG_ON(action->in_flight != 0);
   2.290 +
   2.291 +    /* Disabling IRQ before releasing the desc_lock avoids an IRQ storm. */
   2.292 +    desc->depth   = 1;
   2.293 +    desc->status |= IRQ_DISABLED;
   2.294 +    desc->handler->disable(vector);
   2.295 +
   2.296 +    /*
   2.297 +     * We may have a EOI languishing anywhere in one of the per-CPU
   2.298 +     * EOI stacks. Forcibly flush the stack on every CPU where this might
   2.299 +     * be the case.
   2.300 +     */
   2.301 +    cpu_eoi_map = action->cpu_eoi_map;
   2.302 +    if ( !cpus_empty(cpu_eoi_map) )
   2.303 +    {
   2.304 +        BUG_ON(action->ack_type != ACKTYPE_EOI);
   2.305 +        spin_unlock_irqrestore(&desc->lock, flags);
   2.306 +        on_selected_cpus(cpu_eoi_map, flush_all_pending_eoi, NULL, 1, 1);
   2.307 +        on_selected_cpus(cpu_online_map, flush_ready_eoi, NULL, 1, 1);
   2.308 +        spin_lock_irqsave(&desc->lock, flags);
   2.309 +    }
   2.310 +
   2.311      BUG_ON(!cpus_empty(action->cpu_eoi_map));
   2.312  
   2.313      desc->action = NULL;
   2.314      xfree(action);
   2.315 -    desc->depth   = 1;
   2.316 -    desc->status |= IRQ_DISABLED;
   2.317      desc->status &= ~IRQ_GUEST;
   2.318      desc->handler->shutdown(vector);
   2.319  
   2.320 @@ -543,40 +620,20 @@ static int __init setup_dump_irqs(void)
   2.321  
   2.322  static struct timer end_irq_timer[NR_CPUS];
   2.323  
   2.324 +/*
   2.325 + * force_intack: Forcibly emit all pending EOIs on each CPU every second.
   2.326 + * Mainly useful for debugging or poking lazy guests ISRs.
   2.327 + */
   2.328 +
   2.329  static void end_irq_timeout(void *unused)
   2.330  {
   2.331 -    irq_desc_t         *desc;
   2.332 -    irq_guest_action_t *action;
   2.333 -    cpumask_t           cpu_eoi_map;
   2.334 -    unsigned int        cpu = smp_processor_id();
   2.335 -    int                 sp, vector, i;
   2.336 +    int cpu = smp_processor_id();
   2.337  
   2.338      local_irq_disable();
   2.339 -
   2.340 -    if ( (sp = pending_lapic_eoi_sp(cpu)) == 0 )
   2.341 -    {
   2.342 -        local_irq_enable();
   2.343 -        return;
   2.344 -    }
   2.345 -
   2.346 -    vector = pending_lapic_eoi[cpu][sp-1].vector;
   2.347 -    ASSERT(!pending_lapic_eoi[cpu][sp-1].ready_to_end);
   2.348 -
   2.349 -    desc = &irq_desc[vector];
   2.350 -    spin_lock(&desc->lock);
   2.351 -    action = (irq_guest_action_t *)desc->action;
   2.352 -    ASSERT(action->ack_type == ACKTYPE_LAPIC_EOI);
   2.353 -    ASSERT(desc->status & IRQ_GUEST);
   2.354 -    for ( i = 0; i < action->nr_guests; i++ )
   2.355 -        clear_bit(vector_to_irq(vector), &action->guest[i]->pirq_mask);
   2.356 -    action->in_flight = 0;
   2.357 -    cpu_eoi_map = action->cpu_eoi_map;
   2.358 -    spin_unlock(&desc->lock);
   2.359 -
   2.360 +    flush_all_pending_eoi(NULL);
   2.361      local_irq_enable();
   2.362  
   2.363 -    if ( !cpus_empty(cpu_eoi_map) )
   2.364 -        on_selected_cpus(cpu_eoi_map, end_guest_irq, desc, 1, 0);
   2.365 +    on_selected_cpus(cpu_online_map, flush_ready_eoi, NULL, 1, 0);
   2.366  
   2.367      set_timer(&end_irq_timer[cpu], NOW() + MILLISECS(1000));
   2.368  }