--- /dev/null
+/*
+ * Contains CPU specific branch predictor invalidation sequences
+ *
+ * Copyright (C) 2018 ARM Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+.macro ventry target
+ .rept 31
+ nop
+ .endr
+ b \target
+.endm
+
+.macro vectors target
+ ventry \target + 0x000
+ ventry \target + 0x080
+ ventry \target + 0x100
+ ventry \target + 0x180
+
+ ventry \target + 0x200
+ ventry \target + 0x280
+ ventry \target + 0x300
+ ventry \target + 0x380
+
+ ventry \target + 0x400
+ ventry \target + 0x480
+ ventry \target + 0x500
+ ventry \target + 0x580
+
+ ventry \target + 0x600
+ ventry \target + 0x680
+ ventry \target + 0x700
+ ventry \target + 0x780
+.endm
+
+/*
+ * Populate 4 vector tables. This will cover up to 4 different
+ * micro-architectures in a system.
+ */
+ .align 11
+ENTRY(__bp_harden_hyp_vecs_start)
+ .rept 4
+ vectors hyp_traps_vector
+ .endr
+ENTRY(__bp_harden_hyp_vecs_end)
+
+/*
+ * Local variables:
+ * mode: ASM
+ * indent-tabs-mode: nil
+ * End:
+ */
#include <xen/config.h>
+#include <xen/cpumask.h>
+#include <xen/mm.h>
+#include <xen/sizes.h>
+#include <xen/smp.h>
+#include <xen/spinlock.h>
+#include <xen/vmap.h>
#include <asm/cpufeature.h>
#include <asm/cpuerrata.h>
+/* Hardening Branch predictor code for Arm64 */
+#ifdef CONFIG_ARM64_HARDEN_BRANCH_PREDICTOR
+
+#define VECTOR_TABLE_SIZE SZ_2K
+
+/*
+ * Number of available table vectors (this should be in-sync with
+ * arch/arm64/bpi.S
+ */
+#define NR_BPI_HYP_VECS 4
+
+extern char __bp_harden_hyp_vecs_start[], __bp_harden_hyp_vecs_end[];
+
+/*
+ * Key for each slot. This is used to find whether a specific workaround
+ * had a slot assigned.
+ *
+ * The key is virtual address of the vector workaround
+ */
+static uintptr_t bp_harden_slot_key[NR_BPI_HYP_VECS];
+
+/*
+ * [hyp_vec_start, hyp_vec_end[ corresponds to the first 31 instructions
+ * of each vector. The last (i.e 32th) instruction is used to branch to
+ * the original entry.
+ *
+ * Those instructions will be copied on each vector to harden them.
+ */
+static bool copy_hyp_vect_bpi(unsigned int slot, const char *hyp_vec_start,
+ const char *hyp_vec_end)
+{
+ void *dst_remapped;
+ const void *dst = __bp_harden_hyp_vecs_start + slot * VECTOR_TABLE_SIZE;
+ unsigned int i;
+ mfn_t dst_mfn = _mfn(virt_to_mfn(dst));
+
+ BUG_ON(((hyp_vec_end - hyp_vec_start) / 4) > 31);
+
+ /*
+ * Vectors are part of the text that are mapped read-only. So re-map
+ * the vector table to be able to update vectors.
+ */
+ dst_remapped = __vmap(&dst_mfn,
+ 1UL << get_order_from_bytes(VECTOR_TABLE_SIZE),
+ 1, 1, PAGE_HYPERVISOR, VMAP_DEFAULT);
+ if ( !dst_remapped )
+ return false;
+
+ dst_remapped += (vaddr_t)dst & ~PAGE_MASK;
+
+ for ( i = 0; i < VECTOR_TABLE_SIZE; i += 0x80 )
+ {
+ memcpy(dst_remapped + i, hyp_vec_start, hyp_vec_end - hyp_vec_start);
+ }
+
+ clean_dcache_va_range(dst_remapped, VECTOR_TABLE_SIZE);
+ invalidate_icache();
+
+ vunmap(dst_remapped);
+
+ return true;
+}
+
+static bool __maybe_unused
+install_bp_hardening_vec(const struct arm_cpu_capabilities *entry,
+ const char *hyp_vec_start,
+ const char *hyp_vec_end)
+{
+ static int last_slot = -1;
+ static DEFINE_SPINLOCK(bp_lock);
+ unsigned int i, slot = -1;
+ bool ret = true;
+
+ /*
+ * Enable callbacks are called on every CPU based on the
+ * capabilities. So double-check whether the CPU matches the
+ * entry.
+ */
+ if ( !entry->matches(entry) )
+ return true;
+
+ /*
+ * No need to install hardened vector when the processor has
+ * ID_AA64PRF0_EL1.CSV2 set.
+ */
+ if ( cpu_data[smp_processor_id()].pfr64.csv2 )
+ return true;
+
+ spin_lock(&bp_lock);
+
+ /*
+ * Look up whether the hardening vector had a slot already
+ * assigned.
+ */
+ for ( i = 0; i < 4; i++ )
+ {
+ if ( bp_harden_slot_key[i] == (uintptr_t)hyp_vec_start )
+ {
+ slot = i;
+ break;
+ }
+ }
+
+ if ( slot == -1 )
+ {
+ last_slot++;
+ /* Check we don't overrun the number of slots available. */
+ BUG_ON(NR_BPI_HYP_VECS <= last_slot);
+
+ slot = last_slot;
+ ret = copy_hyp_vect_bpi(slot, hyp_vec_start, hyp_vec_end);
+
+ /* Only update the slot if the copy succeeded. */
+ if ( ret )
+ bp_harden_slot_key[slot] = (uintptr_t)hyp_vec_start;
+ }
+
+ if ( ret )
+ {
+ /* Install the new vector table. */
+ WRITE_SYSREG((vaddr_t)(__bp_harden_hyp_vecs_start + slot * VECTOR_TABLE_SIZE),
+ VBAR_EL2);
+ isb();
+ }
+
+ spin_unlock(&bp_lock);
+
+ return ret;
+}
+
+#endif /* CONFIG_ARM64_HARDEN_BRANCH_PREDICTOR */
+
#define MIDR_RANGE(model, min, max) \
.matches = is_affected_midr_range, \
.midr_model = model, \
projects / xen.git / commitdiff