'-D__OBJECT_LABEL__=$(subst $(BASEDIR)/,,$(CURDIR))/$$@')
$(call as-option-add,CFLAGS,CC,"invpcid (%rax)$$(comma)%rax",-DHAVE_AS_INVPCID)
+# GAS's idea of true is -1. Clang's idea is 1
+$(call as-option-add,CFLAGS,CC,\
+ ".if ((1 > 0) < 0); .error \"\";.endif",,-DHAVE_AS_NEGATIVE_TRUE)
+
CFLAGS += -mno-red-zone -fpic -fno-asynchronous-unwind-tables
# Xen doesn't use SSE interally. If the compiler supports it, also skip the
* APs have less capabilities than the boot processor are not handled.
* Tough. Make sure you disable such features by hand.
*/
-void init_or_livepatch apply_alternatives(const struct alt_instr *start,
- const struct alt_instr *end)
+void init_or_livepatch apply_alternatives(struct alt_instr *start,
+ struct alt_instr *end)
{
- const struct alt_instr *a;
+ struct alt_instr *a, *base;
printk(KERN_INFO "alt table %p -> %p\n", start, end);
* So be careful if you want to change the scan order to any other
* order.
*/
- for ( a = start; a < end; a++ )
+ for ( a = base = start; a < end; a++ )
{
uint8_t *orig = ALT_ORIG_PTR(a);
uint8_t *repl = ALT_REPL_PTR(a);
uint8_t buf[MAX_PATCH_LEN];
+ unsigned int total_len = a->orig_len + a->pad_len;
- BUG_ON(a->repl_len > a->orig_len);
- BUG_ON(a->orig_len > sizeof(buf));
+ BUG_ON(a->repl_len > total_len);
+ BUG_ON(total_len > sizeof(buf));
BUG_ON(a->cpuid >= NCAPINTS * 32);
+ /*
+ * Detect sequences of alt_instr's patching the same origin site, and
+ * keep base pointing at the first alt_instr entry. This is so we can
+ * refer to a single ->priv field for patching decisions. We
+ * deliberately use the alt_instr itself rather than a local variable
+ * in case we end up making multiple passes.
+ *
+ * ->priv being nonzero means that the origin site has already been
+ * modified, and we shouldn't try to optimise the nops again.
+ */
+ if ( ALT_ORIG_PTR(base) != orig )
+ base = a;
+
+ /* If there is no replacement to make, see about optimising the nops. */
if ( !boot_cpu_has(a->cpuid) )
+ {
+ /* Origin site site already touched? Don't nop anything. */
+ if ( base->priv )
+ continue;
+
+ base->priv = 1;
+
+ /* Nothing useful to do? */
+ if ( a->pad_len <= 1 )
+ continue;
+
+ add_nops(buf, a->pad_len);
+ text_poke(orig + a->orig_len, buf, a->pad_len);
continue;
+ }
+
+ base->priv = 1;
memcpy(buf, repl, a->repl_len);
if ( a->repl_len >= 5 && (*buf & 0xfe) == 0xe8 )
*(int32_t *)(buf + 1) += repl - orig;
- add_nops(buf + a->repl_len, a->orig_len - a->repl_len);
- text_poke(orig, buf, a->orig_len);
+ add_nops(buf + a->repl_len, total_len - a->repl_len);
+ text_poke(orig, buf, total_len);
}
}
* enough information for the alternatives patching code to patch an
* instruction. See apply_alternatives().
*/
-.macro altinstruction_entry orig repl feature orig_len repl_len
+.macro altinstruction_entry orig repl feature orig_len repl_len pad_len
.long \orig - .
.long \repl - .
.word \feature
.byte \orig_len
.byte \repl_len
+ .byte \pad_len
+ .byte 0 /* priv */
.endm
-#define decl_orig(insn) .L\@_orig_s: insn; .L\@_orig_e:
+/* GAS's idea of true is -1, while Clang's idea is 1. */
+#ifdef HAVE_AS_NEGATIVE_TRUE
+# define as_true(x) (-(x))
+#else
+# define as_true(x) (x)
+#endif
+
+#define decl_orig(insn, padding) \
+ .L\@_orig_s: insn; .L\@_orig_e: \
+ .L\@_diff = padding; \
+ .skip as_true(.L\@_diff > 0) * .L\@_diff, 0x90; \
+ .L\@_orig_p:
+
#define orig_len (.L\@_orig_e - .L\@_orig_s)
+#define pad_len (.L\@_orig_p - .L\@_orig_e)
+#define total_len (.L\@_orig_p - .L\@_orig_s)
#define decl_repl(insn, nr) .L\@_repl_s\()nr: insn; .L\@_repl_e\()nr:
#define repl_len(nr) (.L\@_repl_e\()nr - .L\@_repl_s\()nr)
+#define as_max(a, b) ((a) ^ (((a) ^ (b)) & -as_true((a) < (b))))
+
.macro ALTERNATIVE oldinstr, newinstr, feature
- decl_orig(\oldinstr)
+ decl_orig(\oldinstr, repl_len(1) - orig_len)
.pushsection .altinstructions, "a", @progbits
altinstruction_entry .L\@_orig_s, .L\@_repl_s1, \feature, \
- orig_len, repl_len(1)
+ orig_len, repl_len(1), pad_len
.section .discard, "a", @progbits
- /* Assembler-time check that \newinstr isn't longer than \oldinstr. */
- .byte 0xff + repl_len(1) - orig_len
+ /*
+ * Assembler-time checks:
+ * - total_len <= 255
+ * - \newinstr <= total_len
+ */
+ .byte total_len
+ .byte 0xff + repl_len(1) - total_len
.section .altinstr_replacement, "ax", @progbits
.endm
.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
- decl_orig(\oldinstr)
+ decl_orig(\oldinstr, as_max(repl_len(1), repl_len(2)) - orig_len)
.pushsection .altinstructions, "a", @progbits
altinstruction_entry .L\@_orig_s, .L\@_repl_s1, \feature1, \
- orig_len, repl_len(1)
+ orig_len, repl_len(1), pad_len
altinstruction_entry .L\@_orig_s, .L\@_repl_s2, \feature2, \
- orig_len, repl_len(2)
+ orig_len, repl_len(2), pad_len
.section .discard, "a", @progbits
- /* Assembler-time check that \newinstr{1,2} aren't longer than \oldinstr. */
- .byte 0xff + repl_len(1) - orig_len
- .byte 0xff + repl_len(2) - orig_len
+ /*
+ * Assembler-time checks:
+ * - total_len <= 255
+ * - \newinstr* <= total_len
+ */
+ .byte total_len
+ .byte 0xff + repl_len(1) - total_len
+ .byte 0xff + repl_len(2) - total_len
.section .altinstr_replacement, "ax", @progbits
.popsection
.endm
+#undef as_max
#undef repl_len
#undef decl_repl
+#undef total_len
+#undef pad_len
#undef orig_len
#undef decl_orig
+#undef as_true
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_ALTERNATIVE_ASM_H_ */
#include <xen/stringify.h>
#include <xen/types.h>
-struct alt_instr {
+struct __packed alt_instr {
int32_t orig_offset; /* original instruction */
int32_t repl_offset; /* offset to replacement instruction */
uint16_t cpuid; /* cpuid bit set for replacement */
uint8_t orig_len; /* length of original instruction */
- uint8_t repl_len; /* length of new instruction, <= instrlen */
+ uint8_t repl_len; /* length of new instruction */
+ uint8_t pad_len; /* length of build-time padding */
+ uint8_t priv; /* Private, for use by apply_alternatives() */
};
#define __ALT_PTR(a,f) ((uint8_t *)((void *)&(a)->f + (a)->f))
extern void add_nops(void *insns, unsigned int len);
/* Similar to alternative_instructions except it can be run with IRQs enabled. */
-extern void apply_alternatives(const struct alt_instr *start,
- const struct alt_instr *end);
+extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
extern void alternative_instructions(void);
-#define OLDINSTR(oldinstr) ".LXEN%=_orig_s:\n\t" oldinstr "\n.LXEN%=_orig_e:\n"
-
#define alt_orig_len "(.LXEN%=_orig_e - .LXEN%=_orig_s)"
+#define alt_pad_len "(.LXEN%=_orig_p - .LXEN%=_orig_e)"
+#define alt_total_len "(.LXEN%=_orig_p - .LXEN%=_orig_s)"
#define alt_repl_s(num) ".LXEN%=_repl_s"#num
#define alt_repl_e(num) ".LXEN%=_repl_e"#num
#define alt_repl_len(num) "(" alt_repl_e(num) " - " alt_repl_s(num) ")"
+/* GAS's idea of true is -1, while Clang's idea is 1. */
+#ifdef HAVE_AS_NEGATIVE_TRUE
+# define AS_TRUE "-"
+#else
+# define AS_TRUE ""
+#endif
+
+#define as_max(a, b) "(("a") ^ ((("a") ^ ("b")) & -("AS_TRUE"(("a") < ("b")))))"
+
+#define OLDINSTR(oldinstr, padding) \
+ ".LXEN%=_orig_s:\n\t" oldinstr "\n .LXEN%=_orig_e:\n\t" \
+ ".LXEN%=_diff = " padding "\n\t" \
+ ".skip "AS_TRUE"(.LXEN%=_diff > 0) * .LXEN%=_diff, 0x90\n\t" \
+ ".LXEN%=_orig_p:\n\t"
+
+#define OLDINSTR_1(oldinstr, n1) \
+ OLDINSTR(oldinstr, alt_repl_len(n1) "-" alt_orig_len)
+
+#define OLDINSTR_2(oldinstr, n1, n2) \
+ OLDINSTR(oldinstr, \
+ as_max((alt_repl_len(n1), \
+ alt_repl_len(n2)) "-" alt_orig_len))
+
#define ALTINSTR_ENTRY(feature, num) \
" .long .LXEN%=_orig_s - .\n" /* label */ \
" .long " alt_repl_s(num)" - .\n" /* new instruction */ \
" .word " __stringify(feature) "\n" /* feature bit */ \
" .byte " alt_orig_len "\n" /* source len */ \
- " .byte " alt_repl_len(num) "\n" /* replacement len */
+ " .byte " alt_repl_len(num) "\n" /* replacement len */ \
+ " .byte " alt_pad_len "\n" /* padding len */ \
+ " .byte 0\n" /* priv */
-#define DISCARD_ENTRY(num) /* repl <= orig */ \
- " .byte 0xff + (" alt_repl_len(num) ") - (" alt_orig_len ")\n"
+#define DISCARD_ENTRY(num) /* repl <= total */ \
+ " .byte 0xff + (" alt_repl_len(num) ") - (" alt_total_len ")\n"
#define ALTINSTR_REPLACEMENT(newinstr, num) /* replacement */ \
alt_repl_s(num)":\n\t" newinstr "\n" alt_repl_e(num) ":\n\t"
/* alternative assembly primitive: */
#define ALTERNATIVE(oldinstr, newinstr, feature) \
- OLDINSTR(oldinstr) \
+ OLDINSTR_1(oldinstr, 1) \
".pushsection .altinstructions, \"a\", @progbits\n" \
ALTINSTR_ENTRY(feature, 1) \
".section .discard, \"a\", @progbits\n" \
+ ".byte " alt_total_len "\n" /* total_len <= 255 */ \
DISCARD_ENTRY(1) \
".section .altinstr_replacement, \"ax\", @progbits\n" \
ALTINSTR_REPLACEMENT(newinstr, 1) \
".popsection\n"
#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
- OLDINSTR(oldinstr) \
+ OLDINSTR_2(oldinstr, 1, 2) \
".pushsection .altinstructions, \"a\", @progbits\n" \
ALTINSTR_ENTRY(feature1, 1) \
ALTINSTR_ENTRY(feature2, 2) \
".section .discard, \"a\", @progbits\n" \
+ ".byte " alt_total_len "\n" /* total_len <= 255 */ \
DISCARD_ENTRY(1) \
DISCARD_ENTRY(2) \
".section .altinstr_replacement, \"ax\", @progbits\n" \
projects / people / andrewcoop / xen.git / commitdiff