xen/evtchn: MAke round-robin scan fairer by snapshotting each l2 word

once only (except for starting l2 word, which we scan in two parts).

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>

diff --git a/drivers/xen/core/evtchn.c b/drivers/xen/core/evtchn.c
index c395ce520139d0150909637bd65b0a219896d6e9..469f078a07eb957fc61280152aaf710ff8308de5 100644 (file)
--- a/drivers/xen/core/evtchn.c
+++ b/drivers/xen/core/evtchn.c
@@ -238,7 +238,7 @@ asmlinkage void evtchn_do_upcall(struct pt_regs *regs)
 {
        unsigned long       l1, l2;
        unsigned long       masked_l1, masked_l2;
-       unsigned int        l1i, l2i, port, count;
+       unsigned int        l1i, l2i, start_l1i, start_l2i, port, count, i;
        int                 irq;
        unsigned int        cpu = smp_processor_id();
        shared_info_t      *s = HYPERVISOR_shared_info;
@@ -261,10 +261,10 @@ asmlinkage void evtchn_do_upcall(struct pt_regs *regs)
 #endif
                l1 = xchg(&vcpu_info->evtchn_pending_sel, 0);
 
-               l1i = per_cpu(current_l1i, cpu);
-               l2i = per_cpu(current_l2i, cpu);
+               start_l1i = l1i = per_cpu(current_l1i, cpu);
+               start_l2i = per_cpu(current_l2i, cpu);
 
-               while (l1 != 0) {
+               for (i = 0; l1 != 0; i++) {
                        masked_l1 = l1 & ((~0UL) << l1i);
                        /* If we masked out all events, wrap to beginning. */
                        if (masked_l1 == 0) {
@@ -273,8 +273,19 @@ asmlinkage void evtchn_do_upcall(struct pt_regs *regs)
                        }
                        l1i = __ffs(masked_l1);
 
+                       l2 = active_evtchns(cpu, s, l1i);
+                       l2i = 0; /* usually scan entire word from start */
+                       if (l1i == start_l1i) {
+                               /* We scan the starting word in two parts. */
+                               if (i == 0)
+                                       /* 1st time: start in the middle */
+                                       l2i = start_l2i;
+                               else
+                                       /* 2nd time: mask bits done already */
+                                       l2 &= (1ul << start_l2i) - 1;
+                       }
+
                        do {
-                               l2 = active_evtchns(cpu, s, l1i);
                                masked_l2 = l2 & ((~0UL) << l2i);
                                if (masked_l2 == 0)
                                        break;
@@ -296,9 +307,8 @@ asmlinkage void evtchn_do_upcall(struct pt_regs *regs)
 
                        } while (l2i != 0);
 
-                       l2 = active_evtchns(cpu, s, l1i);
-                       /* If we handled all ports, clear the selector bit. */
-                       if (l2 == 0)
+                       /* Scan start_l1i twice; all others once. */
+                       if ((l1i != start_l1i) || (i != 0))
                                l1 &= ~(1UL << l1i);
 
                        l1i = (l1i + 1) % BITS_PER_LONG;