rwlock: Allow to scale to 2^31-1 readers on x86.

Also rework to match the 'trylock' style of raw function used for
spinlocks.

Inspired by Jan Beulich's patch to do similar improved scaling.

Signed-off-by: Keir Fraser <keir@xen.org>

diff --git a/xen/common/spinlock.c b/xen/common/spinlock.c
index 2abf89e9b11dbe2aa3130e14a8bf217856870595..a76038c4f27e7fe841a01a91e33f0f42062c637d 100644 (file)
--- a/xen/common/spinlock.c
+++ b/xen/common/spinlock.c
@@ -234,7 +234,11 @@ void _spin_unlock_recursive(spinlock_t *lock)
 void _read_lock(rwlock_t *lock)
 {
     check_lock(&lock->debug);
-    _raw_read_lock(&lock->raw);
+    while ( unlikely(!_raw_read_trylock(&lock->raw)) )
+    {
+        while ( likely(_raw_rw_is_write_locked(&lock->raw)) )
+            cpu_relax();
+    }
     preempt_disable();
 }
 
@@ -243,7 +247,13 @@ void _read_lock_irq(rwlock_t *lock)
     ASSERT(local_irq_is_enabled());
     local_irq_disable();
     check_lock(&lock->debug);
-    _raw_read_lock(&lock->raw);
+    while ( unlikely(!_raw_read_trylock(&lock->raw)) )
+    {
+        local_irq_enable();
+        while ( likely(_raw_rw_is_write_locked(&lock->raw)) )
+            cpu_relax();
+        local_irq_disable();
+    }
     preempt_disable();
 }
 
@@ -252,11 +262,26 @@ unsigned long _read_lock_irqsave(rwlock_t *lock)
     unsigned long flags;
     local_irq_save(flags);
     check_lock(&lock->debug);
-    _raw_read_lock(&lock->raw);
+    while ( unlikely(!_raw_read_trylock(&lock->raw)) )
+    {
+        local_irq_restore(flags);
+        while ( likely(_raw_rw_is_write_locked(&lock->raw)) )
+            cpu_relax();
+        local_irq_save(flags);
+    }
     preempt_disable();
     return flags;
 }
 
+int _read_trylock(rwlock_t *lock)
+{
+    check_lock(&lock->debug);
+    if ( !_raw_read_trylock(&lock->raw) )
+        return 0;
+    preempt_disable();
+    return 1;
+}
+
 void _read_unlock(rwlock_t *lock)
 {
     preempt_enable();
@@ -280,7 +305,11 @@ void _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 void _write_lock(rwlock_t *lock)
 {
     check_lock(&lock->debug);
-    _raw_write_lock(&lock->raw);
+    while ( unlikely(!_raw_write_trylock(&lock->raw)) )
+    {
+        while ( likely(_raw_rw_is_locked(&lock->raw)) )
+            cpu_relax();
+    }
     preempt_disable();
 }
 
@@ -289,7 +318,13 @@ void _write_lock_irq(rwlock_t *lock)
     ASSERT(local_irq_is_enabled());
     local_irq_disable();
     check_lock(&lock->debug);
-    _raw_write_lock(&lock->raw);
+    while ( unlikely(!_raw_write_trylock(&lock->raw)) )
+    {
+        local_irq_enable();
+        while ( likely(_raw_rw_is_locked(&lock->raw)) )
+            cpu_relax();
+        local_irq_disable();
+    }
     preempt_disable();
 }
 
@@ -298,7 +333,13 @@ unsigned long _write_lock_irqsave(rwlock_t *lock)
     unsigned long flags;
     local_irq_save(flags);
     check_lock(&lock->debug);
-    _raw_write_lock(&lock->raw);
+    while ( unlikely(!_raw_write_trylock(&lock->raw)) )
+    {
+        local_irq_restore(flags);
+        while ( likely(_raw_rw_is_locked(&lock->raw)) )
+            cpu_relax();
+        local_irq_save(flags);
+    }
     preempt_disable();
     return flags;
 }

diff --git a/xen/include/asm-ia64/linux-xen/asm/spinlock.h b/xen/include/asm-ia64/linux-xen/asm/spinlock.h
index 0f5311b463ba96758511b70dd71842393efd636a..472d289afb08c79106c3d831d9cf8eee2ad81c68 100644 (file)
--- a/xen/include/asm-ia64/linux-xen/asm/spinlock.h
+++ b/xen/include/asm-ia64/linux-xen/asm/spinlock.h
@@ -35,17 +35,6 @@ typedef struct {
 } raw_rwlock_t;
 #define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { 0, 0 }
 
-#define _raw_read_lock(rw)                                                             \
-do {                                                                                   \
-       raw_rwlock_t *__read_lock_ptr = (rw);                                           \
-                                                                                       \
-       while (unlikely(ia64_fetchadd(1, (int *) __read_lock_ptr, acq) < 0)) {          \
-               ia64_fetchadd(-1, (int *) __read_lock_ptr, rel);                        \
-               while (*(volatile int *)__read_lock_ptr < 0)                            \
-                       cpu_relax();                                                    \
-       }                                                                               \
-} while (0)
-
 #define _raw_read_unlock(rw)                                   \
 do {                                                           \
        raw_rwlock_t *__read_lock_ptr = (rw);                   \
@@ -53,20 +42,6 @@ do {                                                         \
 } while (0)
 
 #ifdef ASM_SUPPORTED
-#define _raw_write_lock(rw)                                                    \
-do {                                                                           \
-       __asm__ __volatile__ (                                                  \
-               "mov ar.ccv = r0\n"                                             \
-               "dep r29 = -1, r0, 31, 1;;\n"                                   \
-               "1:\n"                                                          \
-               "ld4 r2 = [%0];;\n"                                             \
-               "cmp4.eq p0,p7 = r0,r2\n"                                       \
-               "(p7) br.cond.spnt.few 1b \n"                                   \
-               "cmpxchg4.acq r2 = [%0], r29, ar.ccv;;\n"                       \
-               "cmp4.eq p0,p7 = r0, r2\n"                                      \
-               "(p7) br.cond.spnt.few 1b;;\n"                                  \
-               :: "r"(rw) : "ar.ccv", "p7", "r2", "r29", "memory");            \
-} while(0)
 
 #define _raw_write_trylock(rw)                                                 \
 ({                                                                             \
@@ -82,16 +57,6 @@ do {                                                                         \
 
 #else /* !ASM_SUPPORTED */
 
-#define _raw_write_lock(l)                                                             \
-({                                                                                     \
-       __u64 ia64_val, ia64_set_val = ia64_dep_mi(-1, 0, 31, 1);                       \
-       __u32 *ia64_write_lock_ptr = (__u32 *) (l);                                     \
-       do {                                                                            \
-               while (*ia64_write_lock_ptr)                                            \
-                       ia64_barrier();                                                 \
-               ia64_val = ia64_cmpxchg4_acq(ia64_write_lock_ptr, ia64_set_val, 0);     \
-       } while (ia64_val);                                                             \
-})
 
 #define _raw_write_trylock(rw)                                         \
 ({                                                                     \

diff --git a/xen/include/asm-x86/spinlock.h b/xen/include/asm-x86/spinlock.h
index b7cd9455842c3c5b820d69965af35322790d1bb3..5f9fabd3459c403727479e21390b2a340a8c3e80 100644 (file)
--- a/xen/include/asm-x86/spinlock.h
+++ b/xen/include/asm-x86/spinlock.h
@@ -35,51 +35,29 @@ typedef struct {
     volatile int lock;
 } raw_rwlock_t;
 
-#define RW_LOCK_BIAS 0x01000000
-#define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { RW_LOCK_BIAS }
+#define RW_WRITE_BIAS 0x7fffffff
+#define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { 0 }
 
-static always_inline void _raw_read_lock(raw_rwlock_t *rw)
+static always_inline int _raw_read_trylock(raw_rwlock_t *rw)
 {
-    asm volatile (
-        "1:  lock; decl %0         \n"
-        "    jns 3f                \n"
-        "    lock; incl %0         \n"
-        "2:  rep; nop              \n"
-        "    cmpl $1,%0            \n"
-        "    js 2b                 \n"
-        "    jmp 1b                \n"
-        "3:"
-        : "=m" (rw->lock) : : "memory" );
-}
+    bool_t acquired;
 
-static always_inline void _raw_write_lock(raw_rwlock_t *rw)
-{
     asm volatile (
-        "1:  lock; subl %1,%0      \n"
-        "    jz 3f                 \n"
-        "    lock; addl %1,%0      \n"
-        "2:  rep; nop              \n"
-        "    cmpl %1,%0            \n"
-        "    jne 2b                \n"
+        "    lock; decl %0         \n"
+        "    jns 2f                \n"
+        "1:  .subsection 1         \n"
+        "2:  lock; incl %0         \n"
+        "    dec %1                \n"
         "    jmp 1b                \n"
-        "3:"
-        : "=m" (rw->lock) : "i" (RW_LOCK_BIAS) : "memory" );
+        "    .subsection 0         \n"
+        : "=m" (rw->lock), "=r" (acquired) : "1" (1) : "memory" );
+
+    return acquired;
 }
 
 static always_inline int _raw_write_trylock(raw_rwlock_t *rw)
 {
-    int rc;
-
-    asm volatile (
-        "    lock; subl %2,%0      \n"
-        "    jz 1f                 \n"
-        "    lock; addl %2,%0      \n"
-        "    dec %1                \n"
-        "1:"
-        : "=m" (rw->lock), "=r" (rc) : "i" (RW_LOCK_BIAS), "1" (1)
-        : "memory" );
-
-    return rc;
+    return (cmpxchg(&rw->lock, 0, RW_WRITE_BIAS) == 0);
 }
 
 static always_inline void _raw_read_unlock(raw_rwlock_t *rw)
@@ -92,11 +70,11 @@ static always_inline void _raw_read_unlock(raw_rwlock_t *rw)
 static always_inline void _raw_write_unlock(raw_rwlock_t *rw)
 {
     asm volatile (
-        "lock ; addl %1,%0"
-        : "=m" ((rw)->lock) : "i" (RW_LOCK_BIAS) : "memory" );
+        "lock ; subl %1,%0"
+        : "=m" ((rw)->lock) : "i" (RW_WRITE_BIAS) : "memory" );
 }
 
-#define _raw_rw_is_locked(x) ((x)->lock < RW_LOCK_BIAS)
-#define _raw_rw_is_write_locked(x) ((x)->lock <= 0)
+#define _raw_rw_is_locked(x) ((x)->lock != 0)
+#define _raw_rw_is_write_locked(x) ((x)->lock > 0)
 
 #endif /* __ASM_SPINLOCK_H */

diff --git a/xen/include/xen/spinlock.h b/xen/include/xen/spinlock.h
index a2d1ed99bf0625d87d9e3edbad0401537994d05a..6ebe646c19be99009a80a05422ea9c0cfeaa072b 100644 (file)
--- a/xen/include/xen/spinlock.h
+++ b/xen/include/xen/spinlock.h
@@ -157,6 +157,7 @@ unsigned long _read_lock_irqsave(rwlock_t *lock);
 void _read_unlock(rwlock_t *lock);
 void _read_unlock_irq(rwlock_t *lock);
 void _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags);
+int _read_trylock(rwlock_t *lock);
 
 void _write_lock(rwlock_t *lock);
 void _write_lock_irq(rwlock_t *lock);
@@ -210,6 +211,7 @@ int _rw_is_write_locked(rwlock_t *lock);
 #define read_unlock(l)                _read_unlock(l)
 #define read_unlock_irq(l)            _read_unlock_irq(l)
 #define read_unlock_irqrestore(l, f)  _read_unlock_irqrestore(l, f)
+#define read_trylock(l)               _read_trylock(l)
 
 #define write_lock(l)                 _write_lock(l)
 #define write_lock_irq(l)             _write_lock_irq(l)