+++ /dev/null
-/*
- * SPDX-License-Identifier: GPL-2.0-or-later
- * Load/store for 128-bit atomic operations, AArch64 version.
- *
- * Copyright (C) 2018, 2023 Linaro, Ltd.
- *
- * See docs/devel/atomics.rst for discussion about the guarantees each
- * atomic primitive is meant to provide.
- */
-
-#ifndef AARCH64_ATOMIC128_LDST_H
-#define AARCH64_ATOMIC128_LDST_H
-
-#include "host/cpuinfo.h"
-#include "tcg/debug-assert.h"
-
-/*
- * Through gcc 10, aarch64 has no support for 128-bit atomics.
- * Through clang 16, without -march=armv8.4-a, __atomic_load_16
- * is incorrectly expanded to a read-write operation.
- *
- * Anyway, this method allows runtime detection of FEAT_LSE2.
- */
-
-#define HAVE_ATOMIC128_RO (cpuinfo & CPUINFO_LSE2)
-#define HAVE_ATOMIC128_RW 1
-
-static inline Int128 atomic16_read_ro(const Int128 *ptr)
-{
- uint64_t l, h;
-
- tcg_debug_assert(HAVE_ATOMIC128_RO);
- /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */
- asm("ldp %[l], %[h], %[mem]"
- : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr));
-
- return int128_make128(l, h);
-}
-
-static inline Int128 atomic16_read_rw(Int128 *ptr)
-{
- uint64_t l, h;
- uint32_t tmp;
-
- if (cpuinfo & CPUINFO_LSE2) {
- /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */
- asm("ldp %[l], %[h], %[mem]"
- : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr));
- } else {
- /* The load must be paired with the store to guarantee not tearing. */
- asm("0: ldxp %[l], %[h], %[mem]\n\t"
- "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
- "cbnz %w[tmp], 0b"
- : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), [l] "=&r"(l), [h] "=&r"(h));
- }
-
- return int128_make128(l, h);
-}
-
-static inline void atomic16_set(Int128 *ptr, Int128 val)
-{
- uint64_t l = int128_getlo(val), h = int128_gethi(val);
- uint64_t t1, t2;
-
- if (cpuinfo & CPUINFO_LSE2) {
- /* With FEAT_LSE2, 16-byte aligned STP is atomic. */
- asm("stp %[l], %[h], %[mem]"
- : [mem] "=m"(*ptr) : [l] "r"(l), [h] "r"(h));
- } else {
- /* Load into temporaries to acquire the exclusive access lock. */
- asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
- "stxp %w[t1], %[l], %[h], %[mem]\n\t"
- "cbnz %w[t1], 0b"
- : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
- : [l] "r"(l), [h] "r"(h));
- }
-}
-
-#endif /* AARCH64_ATOMIC128_LDST_H */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Load/store for 128-bit atomic operations, AArch64 version.
+ *
+ * Copyright (C) 2018, 2023 Linaro, Ltd.
+ *
+ * See docs/devel/atomics.rst for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef AARCH64_ATOMIC128_LDST_H
+#define AARCH64_ATOMIC128_LDST_H
+
+#include "host/cpuinfo.h"
+#include "tcg/debug-assert.h"
+
+/*
+ * Through gcc 10, aarch64 has no support for 128-bit atomics.
+ * Through clang 16, without -march=armv8.4-a, __atomic_load_16
+ * is incorrectly expanded to a read-write operation.
+ *
+ * Anyway, this method allows runtime detection of FEAT_LSE2.
+ */
+
+#define HAVE_ATOMIC128_RO (cpuinfo & CPUINFO_LSE2)
+#define HAVE_ATOMIC128_RW 1
+
+static inline Int128 atomic16_read_ro(const Int128 *ptr)
+{
+ uint64_t l, h;
+
+ tcg_debug_assert(HAVE_ATOMIC128_RO);
+ /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */
+ asm("ldp %[l], %[h], %[mem]"
+ : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr));
+
+ return int128_make128(l, h);
+}
+
+static inline Int128 atomic16_read_rw(Int128 *ptr)
+{
+ uint64_t l, h;
+ uint32_t tmp;
+
+ if (cpuinfo & CPUINFO_LSE2) {
+ /* With FEAT_LSE2, 16-byte aligned LDP is atomic. */
+ asm("ldp %[l], %[h], %[mem]"
+ : [l] "=r"(l), [h] "=r"(h) : [mem] "m"(*ptr));
+ } else {
+ /* The load must be paired with the store to guarantee not tearing. */
+ asm("0: ldxp %[l], %[h], %[mem]\n\t"
+ "stxp %w[tmp], %[l], %[h], %[mem]\n\t"
+ "cbnz %w[tmp], 0b"
+ : [mem] "+m"(*ptr), [tmp] "=&r"(tmp), [l] "=&r"(l), [h] "=&r"(h));
+ }
+
+ return int128_make128(l, h);
+}
+
+static inline void atomic16_set(Int128 *ptr, Int128 val)
+{
+ uint64_t l = int128_getlo(val), h = int128_gethi(val);
+ uint64_t t1, t2;
+
+ if (cpuinfo & CPUINFO_LSE2) {
+ /* With FEAT_LSE2, 16-byte aligned STP is atomic. */
+ asm("stp %[l], %[h], %[mem]"
+ : [mem] "=m"(*ptr) : [l] "r"(l), [h] "r"(h));
+ } else {
+ /* Load into temporaries to acquire the exclusive access lock. */
+ asm("0: ldxp %[t1], %[t2], %[mem]\n\t"
+ "stxp %w[t1], %[l], %[h], %[mem]\n\t"
+ "cbnz %w[t1], 0b"
+ : [mem] "+m"(*ptr), [t1] "=&r"(t1), [t2] "=&r"(t2)
+ : [l] "r"(l), [h] "r"(h));
+ }
+}
+
+#endif /* AARCH64_ATOMIC128_LDST_H */
+++ /dev/null
-/*
- * SPDX-License-Identifier: GPL-2.0-or-later
- * Load/store for 128-bit atomic operations, generic version.
- *
- * Copyright (C) 2018, 2023 Linaro, Ltd.
- *
- * See docs/devel/atomics.rst for discussion about the guarantees each
- * atomic primitive is meant to provide.
- */
-
-#ifndef HOST_ATOMIC128_LDST_H
-#define HOST_ATOMIC128_LDST_H
-
-#if defined(CONFIG_ATOMIC128)
-# define HAVE_ATOMIC128_RO 1
-# define HAVE_ATOMIC128_RW 1
-
-static inline Int128 ATTRIBUTE_ATOMIC128_OPT
-atomic16_read_ro(const Int128 *ptr)
-{
- const __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
- Int128Alias r;
-
- r.i = qatomic_read__nocheck(ptr_align);
- return r.s;
-}
-
-static inline Int128 ATTRIBUTE_ATOMIC128_OPT
-atomic16_read_rw(Int128 *ptr)
-{
- return atomic16_read_ro(ptr);
-}
-
-static inline void ATTRIBUTE_ATOMIC128_OPT
-atomic16_set(Int128 *ptr, Int128 val)
-{
- __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
- Int128Alias v;
-
- v.s = val;
- qatomic_set__nocheck(ptr_align, v.i);
-}
-
-#elif defined(CONFIG_CMPXCHG128)
-# define HAVE_ATOMIC128_RO 0
-# define HAVE_ATOMIC128_RW 1
-
-Int128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr);
-
-static inline Int128 ATTRIBUTE_ATOMIC128_OPT
-atomic16_read_rw(Int128 *ptr)
-{
- /* Maybe replace 0 with 0, returning the old value. */
- Int128 z = int128_make64(0);
- return atomic16_cmpxchg(ptr, z, z);
-}
-
-static inline void ATTRIBUTE_ATOMIC128_OPT
-atomic16_set(Int128 *ptr, Int128 val)
-{
- Int128Aligned *ptr_align = __builtin_assume_aligned(ptr, 16);
- __int128_t old;
- Int128Alias new;
-
- new.s = val;
- do {
- old = *ptr_align;
- } while (!__sync_bool_compare_and_swap_16(ptr_align, old, new.i));
-}
-
-#else
-# define HAVE_ATOMIC128_RO 0
-# define HAVE_ATOMIC128_RW 0
-
-/* Fallback definitions that must be optimized away, or error. */
-Int128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr);
-Int128 QEMU_ERROR("unsupported atomic") atomic16_read_rw(Int128 *ptr);
-void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
-#endif
-
-#endif /* HOST_ATOMIC128_LDST_H */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Load/store for 128-bit atomic operations, generic version.
+ *
+ * Copyright (C) 2018, 2023 Linaro, Ltd.
+ *
+ * See docs/devel/atomics.rst for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef HOST_ATOMIC128_LDST_H
+#define HOST_ATOMIC128_LDST_H
+
+#if defined(CONFIG_ATOMIC128)
+# define HAVE_ATOMIC128_RO 1
+# define HAVE_ATOMIC128_RW 1
+
+static inline Int128 ATTRIBUTE_ATOMIC128_OPT
+atomic16_read_ro(const Int128 *ptr)
+{
+ const __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ Int128Alias r;
+
+ r.i = qatomic_read__nocheck(ptr_align);
+ return r.s;
+}
+
+static inline Int128 ATTRIBUTE_ATOMIC128_OPT
+atomic16_read_rw(Int128 *ptr)
+{
+ return atomic16_read_ro(ptr);
+}
+
+static inline void ATTRIBUTE_ATOMIC128_OPT
+atomic16_set(Int128 *ptr, Int128 val)
+{
+ __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ Int128Alias v;
+
+ v.s = val;
+ qatomic_set__nocheck(ptr_align, v.i);
+}
+
+#elif defined(CONFIG_CMPXCHG128)
+# define HAVE_ATOMIC128_RO 0
+# define HAVE_ATOMIC128_RW 1
+
+Int128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr);
+
+static inline Int128 ATTRIBUTE_ATOMIC128_OPT
+atomic16_read_rw(Int128 *ptr)
+{
+ /* Maybe replace 0 with 0, returning the old value. */
+ Int128 z = int128_make64(0);
+ return atomic16_cmpxchg(ptr, z, z);
+}
+
+static inline void ATTRIBUTE_ATOMIC128_OPT
+atomic16_set(Int128 *ptr, Int128 val)
+{
+ Int128Aligned *ptr_align = __builtin_assume_aligned(ptr, 16);
+ __int128_t old;
+ Int128Alias new;
+
+ new.s = val;
+ do {
+ old = *ptr_align;
+ } while (!__sync_bool_compare_and_swap_16(ptr_align, old, new.i));
+}
+
+#else
+# define HAVE_ATOMIC128_RO 0
+# define HAVE_ATOMIC128_RW 0
+
+/* Fallback definitions that must be optimized away, or error. */
+Int128 QEMU_ERROR("unsupported atomic") atomic16_read_ro(const Int128 *ptr);
+Int128 QEMU_ERROR("unsupported atomic") atomic16_read_rw(Int128 *ptr);
+void QEMU_ERROR("unsupported atomic") atomic16_set(Int128 *ptr, Int128 val);
+#endif
+
+#endif /* HOST_ATOMIC128_LDST_H */
+++ /dev/null
-/*
- * SPDX-License-Identifier: GPL-2.0-or-later
- * Load/store for 128-bit atomic operations, LoongArch version.
- *
- * See docs/devel/atomics.rst for discussion about the guarantees each
- * atomic primitive is meant to provide.
- */
-
-#ifndef LOONGARCH_ATOMIC128_LDST_H
-#define LOONGARCH_ATOMIC128_LDST_H
-
-#include "host/cpuinfo.h"
-#include "tcg/debug-assert.h"
-
-#define HAVE_ATOMIC128_RO likely(cpuinfo & CPUINFO_LSX)
-#define HAVE_ATOMIC128_RW HAVE_ATOMIC128_RO
-
-/*
- * As of gcc 13 and clang 16, there is no compiler support for LSX at all.
- * Use inline assembly throughout.
- */
-
-static inline Int128 atomic16_read_ro(const Int128 *ptr)
-{
- uint64_t l, h;
-
- tcg_debug_assert(HAVE_ATOMIC128_RO);
- asm("vld $vr0, %2, 0\n\t"
- "vpickve2gr.d %0, $vr0, 0\n\t"
- "vpickve2gr.d %1, $vr0, 1"
- : "=r"(l), "=r"(h) : "r"(ptr), "m"(*ptr) : "f0");
-
- return int128_make128(l, h);
-}
-
-static inline Int128 atomic16_read_rw(Int128 *ptr)
-{
- return atomic16_read_ro(ptr);
-}
-
-static inline void atomic16_set(Int128 *ptr, Int128 val)
-{
- uint64_t l = int128_getlo(val), h = int128_gethi(val);
-
- tcg_debug_assert(HAVE_ATOMIC128_RW);
- asm("vinsgr2vr.d $vr0, %1, 0\n\t"
- "vinsgr2vr.d $vr0, %2, 1\n\t"
- "vst $vr0, %3, 0"
- : "=m"(*ptr) : "r"(l), "r"(h), "r"(ptr) : "f0");
-}
-
-#endif /* LOONGARCH_ATOMIC128_LDST_H */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Load/store for 128-bit atomic operations, LoongArch version.
+ *
+ * See docs/devel/atomics.rst for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef LOONGARCH_ATOMIC128_LDST_H
+#define LOONGARCH_ATOMIC128_LDST_H
+
+#include "host/cpuinfo.h"
+#include "tcg/debug-assert.h"
+
+#define HAVE_ATOMIC128_RO likely(cpuinfo & CPUINFO_LSX)
+#define HAVE_ATOMIC128_RW HAVE_ATOMIC128_RO
+
+/*
+ * As of gcc 13 and clang 16, there is no compiler support for LSX at all.
+ * Use inline assembly throughout.
+ */
+
+static inline Int128 atomic16_read_ro(const Int128 *ptr)
+{
+ uint64_t l, h;
+
+ tcg_debug_assert(HAVE_ATOMIC128_RO);
+ asm("vld $vr0, %2, 0\n\t"
+ "vpickve2gr.d %0, $vr0, 0\n\t"
+ "vpickve2gr.d %1, $vr0, 1"
+ : "=r"(l), "=r"(h) : "r"(ptr), "m"(*ptr) : "f0");
+
+ return int128_make128(l, h);
+}
+
+static inline Int128 atomic16_read_rw(Int128 *ptr)
+{
+ return atomic16_read_ro(ptr);
+}
+
+static inline void atomic16_set(Int128 *ptr, Int128 val)
+{
+ uint64_t l = int128_getlo(val), h = int128_gethi(val);
+
+ tcg_debug_assert(HAVE_ATOMIC128_RW);
+ asm("vinsgr2vr.d $vr0, %1, 0\n\t"
+ "vinsgr2vr.d $vr0, %2, 1\n\t"
+ "vst $vr0, %3, 0"
+ : "=m"(*ptr) : "r"(l), "r"(h), "r"(ptr) : "f0");
+}
+
+#endif /* LOONGARCH_ATOMIC128_LDST_H */
+++ /dev/null
-/*
- * SPDX-License-Identifier: GPL-2.0-or-later
- * Load/store for 128-bit atomic operations, x86_64 version.
- *
- * Copyright (C) 2023 Linaro, Ltd.
- *
- * See docs/devel/atomics.rst for discussion about the guarantees each
- * atomic primitive is meant to provide.
- */
-
-#ifndef X86_64_ATOMIC128_LDST_H
-#define X86_64_ATOMIC128_LDST_H
-
-#ifdef CONFIG_INT128_TYPE
-#include "host/cpuinfo.h"
-#include "tcg/debug-assert.h"
-#include <immintrin.h>
-
-typedef union {
- __m128i v;
- __int128_t i;
- Int128 s;
-} X86Int128Union;
-
-/*
- * Through clang 16, with -mcx16, __atomic_load_n is incorrectly
- * expanded to a read-write operation: lock cmpxchg16b.
- */
-
-#define HAVE_ATOMIC128_RO likely(cpuinfo & CPUINFO_ATOMIC_VMOVDQA)
-#define HAVE_ATOMIC128_RW 1
-
-static inline Int128 atomic16_read_ro(const Int128 *ptr)
-{
- X86Int128Union r;
-
- tcg_debug_assert(HAVE_ATOMIC128_RO);
- asm("vmovdqa %1, %0" : "=x" (r.v) : "m" (*ptr));
-
- return r.s;
-}
-
-static inline Int128 atomic16_read_rw(Int128 *ptr)
-{
- __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
- X86Int128Union r;
-
- if (HAVE_ATOMIC128_RO) {
- asm("vmovdqa %1, %0" : "=x" (r.v) : "m" (*ptr_align));
- } else {
- r.i = __sync_val_compare_and_swap_16(ptr_align, 0, 0);
- }
- return r.s;
-}
-
-static inline void atomic16_set(Int128 *ptr, Int128 val)
-{
- __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
- X86Int128Union new = { .s = val };
-
- if (HAVE_ATOMIC128_RO) {
- asm("vmovdqa %1, %0" : "=m"(*ptr_align) : "x" (new.v));
- } else {
- __int128_t old;
- do {
- old = *ptr_align;
- } while (!__sync_bool_compare_and_swap_16(ptr_align, old, new.i));
- }
-}
-#else
-/* Provide QEMU_ERROR stubs. */
-#include "host/include/generic/host/atomic128-ldst.h"
-#endif
-
-#endif /* X86_64_ATOMIC128_LDST_H */
--- /dev/null
+/*
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ * Load/store for 128-bit atomic operations, x86_64 version.
+ *
+ * Copyright (C) 2023 Linaro, Ltd.
+ *
+ * See docs/devel/atomics.rst for discussion about the guarantees each
+ * atomic primitive is meant to provide.
+ */
+
+#ifndef X86_64_ATOMIC128_LDST_H
+#define X86_64_ATOMIC128_LDST_H
+
+#ifdef CONFIG_INT128_TYPE
+#include "host/cpuinfo.h"
+#include "tcg/debug-assert.h"
+#include <immintrin.h>
+
+typedef union {
+ __m128i v;
+ __int128_t i;
+ Int128 s;
+} X86Int128Union;
+
+/*
+ * Through clang 16, with -mcx16, __atomic_load_n is incorrectly
+ * expanded to a read-write operation: lock cmpxchg16b.
+ */
+
+#define HAVE_ATOMIC128_RO likely(cpuinfo & CPUINFO_ATOMIC_VMOVDQA)
+#define HAVE_ATOMIC128_RW 1
+
+static inline Int128 atomic16_read_ro(const Int128 *ptr)
+{
+ X86Int128Union r;
+
+ tcg_debug_assert(HAVE_ATOMIC128_RO);
+ asm("vmovdqa %1, %0" : "=x" (r.v) : "m" (*ptr));
+
+ return r.s;
+}
+
+static inline Int128 atomic16_read_rw(Int128 *ptr)
+{
+ __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ X86Int128Union r;
+
+ if (HAVE_ATOMIC128_RO) {
+ asm("vmovdqa %1, %0" : "=x" (r.v) : "m" (*ptr_align));
+ } else {
+ r.i = __sync_val_compare_and_swap_16(ptr_align, 0, 0);
+ }
+ return r.s;
+}
+
+static inline void atomic16_set(Int128 *ptr, Int128 val)
+{
+ __int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
+ X86Int128Union new = { .s = val };
+
+ if (HAVE_ATOMIC128_RO) {
+ asm("vmovdqa %1, %0" : "=m"(*ptr_align) : "x" (new.v));
+ } else {
+ __int128_t old;
+ do {
+ old = *ptr_align;
+ } while (!__sync_bool_compare_and_swap_16(ptr_align, old, new.i));
+ }
+}
+#else
+/* Provide QEMU_ERROR stubs. */
+#include "host/include/generic/host/atomic128-ldst.h.inc"
+#endif
+
+#endif /* X86_64_ATOMIC128_LDST_H */
#define X86_64_LOAD_EXTRACT_AL16_AL8_H
#ifdef CONFIG_INT128_TYPE
-#include "host/atomic128-ldst.h"
+#include "host/atomic128-ldst.h.inc"
/**
* load_atom_extract_al16_or_al8:
*/
#include "host/atomic128-cas.h.inc"
-#include "host/atomic128-ldst.h"
+#include "host/atomic128-ldst.h.inc"
#endif /* QEMU_ATOMIC128_H */
projects / people / aperard / qemu-dm.git / commitdiff