x86/altcall: Switch to simpler scheme

With all the infrastructure in place, switch from using ALTERNATIVE() to
simply populating .alt_call_sites.

Before, _apply_alternatives() would devirtualise in two passes; the first
being opportunistic, and the second (signified by the force parameter) sealing
any call with a still-NULL function pointer.

Now, all devirtualising is performed together, at the point in time of the
second pass previously.  The call to seal_endbr64() needs delaying until after
apply_alt_calls() is complete, or we have a narrow window with real indirect
branches and no ENDBR64 instructions.

Under the hood, the following changes are happening:

  Section                Old size   New size   Change (%)
  .alt_call_sites               0    0x00730   +0x0730
  .altinstructions        0x1350a    0x11fe0   -0x152a (-7%)
  .altinstr_replacement   0x015f2    0x00e35   -0x07bd (-23%)

The changes aren't quite equal because inlining is affected by the smaller
asm() block.  Nevertheless, the metadata is held in 1/3 of the space, and
there are no CALL instructions held in the replacement section any more.

No functional change.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

diff --git a/xen/arch/x86/alternative.c b/xen/arch/x86/alternative.c
index 22af224f08f7035d56625daff207a2a87e0aaf05..047bfc6e424be2a893514797226c44e5361f97c6 100644 (file)
--- a/xen/arch/x86/alternative.c
+++ b/xen/arch/x86/alternative.c
@@ -378,13 +378,6 @@ static int init_or_livepatch _apply_alternatives(struct alt_instr *start,
         text_poke(orig, buf, total_len);
     }
 
-    /*
-     * Clobber endbr64 instructions now that altcall has finished optimising
-     * all indirect branches to direct ones.
-     */
-    if ( force && system_state < SYS_STATE_active )
-        seal_endbr64();
-
     return 0;
 }
 
@@ -533,6 +526,8 @@ static int __init cf_check nmi_apply_alternatives(
             rc = apply_alt_calls(__alt_call_sites_start, __alt_call_sites_end);
             if ( rc )
                 panic("Unable to apply alternative calls: %d\n", rc);
+
+            seal_endbr64();
         }
 
         /*

diff --git a/xen/arch/x86/include/asm/alternative-call.h b/xen/arch/x86/include/asm/alternative-call.h
index 49a04a7cc45b6de7d63bda3b281ac2d24bcc4780..bbc49a5274d9424392d57cb3dfa07d55126d4197 100644 (file)
--- a/xen/arch/x86/include/asm/alternative-call.h
+++ b/xen/arch/x86/include/asm/alternative-call.h
@@ -2,7 +2,8 @@
 #ifndef X86_ALTERNATIVE_CALL_H
 #define X86_ALTERNATIVE_CALL_H
 
-#include <asm/alternative.h>
+#include <xen/macros.h>
+#include <xen/stdint.h>
 
 /* Simply the relative position of the source call. */
 struct alt_call {
@@ -86,8 +87,10 @@ struct alt_call {
     rettype ret_;                                                  \
     register unsigned long r10_ asm("r10");                        \
     register unsigned long r11_ asm("r11");                        \
-    asm volatile (ALTERNATIVE("call *%c[addr](%%rip)", "call .",   \
-                              X86_FEATURE_ALWAYS)                  \
+    asm volatile ("1: call *%c[addr](%%rip)\n\t"                   \
+                  ".pushsection .alt_call_sites, \"a\", @progbits\n\t"  \
+                  ".long 1b - .\n\t"                               \
+                  ".popsection"                                    \
                   : ALT_CALL ## n ## _OUT, "=a" (ret_),            \
                     "=r" (r10_), "=r" (r11_) ASM_CALL_CONSTRAINT   \
                   : [addr] "i" (&(func)), "g" (func)               \