riscv/barrier: Define RISCV_FULL_BARRIER

Introduce RISCV_FULL_BARRIER and use in arch_atomic* function.
like RISCV_ACQUIRE_BARRIER and RISCV_RELEASE_BARRIER, the fence
instruction can be eliminated When SMP is not enabled.

Signed-off-by: Eric Chan <ericchancf@google.com>
Reviewed-by: Andrea Parri <parri.andrea@gmail.com>
Reviewed-by: Samuel Holland <samuel.holland@sifive.com>
Tested-by: Samuel Holland <samuel.holland@sifive.com>
Link: https://lore.kernel.org/r/20240217131302.3668481-1-ericchancf@google.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index f5dfef6c2153f189288d3a4f3fa7a1d5d1b5021b..31e6e2e7cc181456fdd3233bbafc37bb175bf6f8 100644 (file)
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -207,7 +207,7 @@ static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int
                "       add      %[rc], %[p], %[a]\n"
                "       sc.w.rl  %[rc], %[rc], %[c]\n"
                "       bnez     %[rc], 0b\n"
-               "       fence    rw, rw\n"
+               RISCV_FULL_BARRIER
                "1:\n"
                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
                : [a]"r" (a), [u]"r" (u)
@@ -228,7 +228,7 @@ static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a,
                "       add      %[rc], %[p], %[a]\n"
                "       sc.d.rl  %[rc], %[rc], %[c]\n"
                "       bnez     %[rc], 0b\n"
-               "       fence    rw, rw\n"
+               RISCV_FULL_BARRIER
                "1:\n"
                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
                : [a]"r" (a), [u]"r" (u)
@@ -248,7 +248,7 @@ static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)
                "       addi      %[rc], %[p], 1\n"
                "       sc.w.rl   %[rc], %[rc], %[c]\n"
                "       bnez      %[rc], 0b\n"
-               "       fence     rw, rw\n"
+               RISCV_FULL_BARRIER
                "1:\n"
                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
                :
@@ -268,7 +268,7 @@ static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v)
                "       addi      %[rc], %[p], -1\n"
                "       sc.w.rl   %[rc], %[rc], %[c]\n"
                "       bnez      %[rc], 0b\n"
-               "       fence     rw, rw\n"
+               RISCV_FULL_BARRIER
                "1:\n"
                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
                :
@@ -288,7 +288,7 @@ static __always_inline int arch_atomic_dec_if_positive(atomic_t *v)
                "       bltz     %[rc], 1f\n"
                "       sc.w.rl  %[rc], %[rc], %[c]\n"
                "       bnez     %[rc], 0b\n"
-               "       fence    rw, rw\n"
+               RISCV_FULL_BARRIER
                "1:\n"
                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
                :
@@ -310,7 +310,7 @@ static __always_inline bool arch_atomic64_inc_unless_negative(atomic64_t *v)
                "       addi      %[rc], %[p], 1\n"
                "       sc.d.rl   %[rc], %[rc], %[c]\n"
                "       bnez      %[rc], 0b\n"
-               "       fence     rw, rw\n"
+               RISCV_FULL_BARRIER
                "1:\n"
                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
                :
@@ -331,7 +331,7 @@ static __always_inline bool arch_atomic64_dec_unless_positive(atomic64_t *v)
                "       addi      %[rc], %[p], -1\n"
                "       sc.d.rl   %[rc], %[rc], %[c]\n"
                "       bnez      %[rc], 0b\n"
-               "       fence     rw, rw\n"
+               RISCV_FULL_BARRIER
                "1:\n"
                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
                :
@@ -352,7 +352,7 @@ static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
                "       bltz     %[rc], 1f\n"
                "       sc.d.rl  %[rc], %[rc], %[c]\n"
                "       bnez     %[rc], 0b\n"
-               "       fence    rw, rw\n"
+               RISCV_FULL_BARRIER
                "1:\n"
                : [p]"=&r" (prev), [rc]"=&r" (rc), [c]"+A" (v->counter)
                :

diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
index 2f4726d3cfcc25a805adacc0b0e85c4ff8bcc220..a608e4d1a0a4110f3ae623ce9f3e9c27ace7b25e 100644 (file)
--- a/arch/riscv/include/asm/cmpxchg.h
+++ b/arch/riscv/include/asm/cmpxchg.h
@@ -313,7 +313,7 @@
                        "       bne  %0, %z3, 1f\n"                     \
                        "       sc.w.rl %1, %z4, %2\n"                  \
                        "       bnez %1, 0b\n"                          \
-                       "       fence rw, rw\n"                         \
+                       RISCV_FULL_BARRIER                              \
                        "1:\n"                                          \
                        : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
                        : "rJ" ((long)__old), "rJ" (__new)              \
@@ -325,7 +325,7 @@
                        "       bne %0, %z3, 1f\n"                      \
                        "       sc.d.rl %1, %z4, %2\n"                  \
                        "       bnez %1, 0b\n"                          \
-                       "       fence rw, rw\n"                         \
+                       RISCV_FULL_BARRIER                              \
                        "1:\n"                                          \
                        : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr)    \
                        : "rJ" (__old), "rJ" (__new)                    \

diff --git a/arch/riscv/include/asm/fence.h b/arch/riscv/include/asm/fence.h
index 2b443a3a487f3cdc509b5f80c7c343065ae85ef7..6c26c44dfcd625bfcf80a4afe359cb15c1a84911 100644 (file)
--- a/arch/riscv/include/asm/fence.h
+++ b/arch/riscv/include/asm/fence.h
@@ -4,9 +4,11 @@
 #ifdef CONFIG_SMP
 #define RISCV_ACQUIRE_BARRIER          "\tfence r , rw\n"
 #define RISCV_RELEASE_BARRIER          "\tfence rw,  w\n"
+#define RISCV_FULL_BARRIER             "\tfence rw, rw\n"
 #else
 #define RISCV_ACQUIRE_BARRIER
 #define RISCV_RELEASE_BARRIER
+#define RISCV_FULL_BARRIER
 #endif
 
 #endif /* _ASM_RISCV_FENCE_H */