irqchip, gicv3: Workaround for Cavium ThunderX erratum 23154

This patch implements Cavium ThunderX erratum 23154.

The gicv3 of ThunderX requires a modified version for reading the IAR
status to ensure data synchronization. Since this is in the fast-path
and called with each interrupt, runtime patching is used using jump
label patching for smallest overhead (no-op). This is the same
technique as used for tracepoints.

v4:
 * simplify code to only use cpus_have_cap() in gicv3_enable_quirks()

v3:
 * fix erratum to be dependend from midr
 * use arm64 errata framework

v2:
 * implement code in a single asm() to keep instruction sequence
 * added comment to the code that explains the erratum
 * apply workaround also if running as guest, thus check MIDR

Signed-off-by: Robert Richter <rrichter@cavium.com>
Signed-off-by: Vadim Lomovtsev <Vadim.Lomovtsev@caviumnetworks.com>

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 318175f62c24d0ec4df68a555d02f2bd2a920cfc..1700d96cae14062a6188102833770f75fba44062 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -417,6 +417,17 @@ config ARM64_ERRATUM_845719
 
          If unsure, say Y.
 
+config CAVIUM_ERRATUM_23154
+       bool "Cavium erratum 23154: Access to ICC_IAR1_EL1 is not sync'ed"
+       depends on ARCH_THUNDER
+       default y
+       help
+         The gicv3 of ThunderX requires a modified version for
+         reading the IAR status to ensure data synchronization
+         (access to icc_iar1_el1 is not sync'ed before and after).
+
+         If unsure, say Y.
+
 endmenu
 
 

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index c1044218a63a1598051d3d02b58d3241ec810cc1..2a5e4c163ee55b3418e109b18ccfe5aa5ad177a8 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -25,8 +25,9 @@
 #define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE   1
 #define ARM64_WORKAROUND_845719                        2
 #define ARM64_HAS_SYSREG_GIC_CPUIF             3
+#define ARM64_WORKAROUND_CAVIUM_23154          4
 
-#define ARM64_NCAPS                            4
+#define ARM64_NCAPS                            5
 
 #ifndef __ASSEMBLY__
 

diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index a84ec605bed8190ed90f6a47e09315b21327875b..f6a0ae40a90272e0fa0119770be1d8c4b4b67bc2 100644 (file)
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -62,15 +62,18 @@
        (0xf                    << MIDR_ARCHITECTURE_SHIFT) | \
        ((partnum)              << MIDR_PARTNUM_SHIFT))
 
-#define ARM_CPU_IMP_ARM                0x41
-#define ARM_CPU_IMP_APM                0x50
+#define ARM_CPU_IMP_ARM                        0x41
+#define ARM_CPU_IMP_APM                        0x50
+#define ARM_CPU_IMP_CAVIUM             0x43
 
-#define ARM_CPU_PART_AEM_V8    0xD0F
-#define ARM_CPU_PART_FOUNDATION        0xD00
-#define ARM_CPU_PART_CORTEX_A57        0xD07
-#define ARM_CPU_PART_CORTEX_A53        0xD03
+#define ARM_CPU_PART_AEM_V8            0xD0F
+#define ARM_CPU_PART_FOUNDATION                0xD00
+#define ARM_CPU_PART_CORTEX_A57                0xD07
+#define ARM_CPU_PART_CORTEX_A53                0xD03
 
-#define APM_CPU_PART_POTENZA   0x000
+#define APM_CPU_PART_POTENZA           0x000
+
+#define CAVIUM_CPU_PART_THUNDERX       0x0A1
 
 #define ID_AA64MMFR0_BIGENDEL0_SHIFT   16
 #define ID_AA64MMFR0_BIGENDEL0_MASK    (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT)

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 6ffd914385609d0aab875813e4e9e8b690976421..574450c257a4d038e17d4eb8a47b954b20b2b9bb 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -23,6 +23,7 @@
 
 #define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
 #define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
+#define MIDR_THUNDERX  MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
 
 #define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
                        MIDR_ARCHITECTURE_MASK)
@@ -81,6 +82,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
                .capability = ARM64_WORKAROUND_845719,
                MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04),
        },
+#endif
+#ifdef CONFIG_CAVIUM_ERRATUM_23154
+       {
+       /* Cavium ThunderX, pass 1.x */
+               .desc = "Cavium erratum 23154",
+               .capability = ARM64_WORKAROUND_CAVIUM_23154,
+               MIDR_RANGE(MIDR_THUNDERX, 0x00, 0x01),
+       },
 #endif
        {
        }

diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index c52f7ba205b4c872205323868ecf0349f8174b20..4211c39b8744fa2641392ae47a342cd9476d2611 100644 (file)
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -107,7 +107,7 @@ static void gic_redist_wait_for_rwp(void)
 }
 
 /* Low level accessors */
-static u64 __maybe_unused gic_read_iar(void)
+static u64 gic_read_iar_common(void)
 {
        u64 irqstat;
 
@@ -115,6 +115,38 @@ static u64 __maybe_unused gic_read_iar(void)
        return irqstat;
 }
 
+/*
+ * Cavium ThunderX erratum 23154
+ *
+ * The gicv3 of ThunderX requires a modified version for reading the
+ * IAR status to ensure data synchronization (access to icc_iar1_el1
+ * is not sync'ed before and after).
+ */
+static u64 gic_read_iar_cavium_thunderx(void)
+{
+       u64 irqstat;
+
+       asm volatile(
+               "nop;nop;nop;nop\n\t"
+               "nop;nop;nop;nop\n\t"
+               "mrs_s %0, " __stringify(ICC_IAR1_EL1) "\n\t"
+               "nop;nop;nop;nop"
+               : "=r" (irqstat));
+       mb();
+
+       return irqstat;
+}
+
+struct static_key is_cavium_thunderx = STATIC_KEY_INIT_FALSE;
+
+static u64 __maybe_unused gic_read_iar(void)
+{
+       if (static_key_false(&is_cavium_thunderx))
+               return gic_read_iar_common();
+       else
+               return gic_read_iar_cavium_thunderx();
+}
+
 static void __maybe_unused gic_write_pmr(u64 val)
 {
        asm volatile("msr_s " __stringify(ICC_PMR_EL1) ", %0" : : "r" (val));
@@ -766,6 +798,12 @@ static const struct irq_domain_ops gic_irq_domain_ops = {
        .free = gic_irq_domain_free,
 };
 
+static void gicv3_enable_quirks(void)
+{
+       if (cpus_have_cap(ARM64_WORKAROUND_CAVIUM_23154))
+               static_key_slow_inc(&is_cavium_thunderx);
+}
+
 static int __init gic_of_init(struct device_node *node, struct device_node *parent)
 {
        void __iomem *dist_base;
@@ -825,6 +863,8 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare
        gic_data.nr_redist_regions = nr_redist_regions;
        gic_data.redist_stride = redist_stride;
 
+       gicv3_enable_quirks();
+
        /*
         * Find out how many interrupts are supported.
         * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)