x86/smt: Support for enabling/disabling SMT at runtime

Currently, a user can in principle combine the output of `xl info -n`, the
APCI tables, and some manual CPUID data to figure out which CPU numbers to
feed into `xen-hptool cpu-offline` to effectively disable SMT at runtime.

A more convenient option is to teach Xen how to perform this action.

Extend XEN_SYSCTL_cpu_hotplug with two new operations.  Introduce a new
smt_up_down_helper() which wraps the cpu_{up,down}_helper() helpers with logic
which understands siblings based on their APIC_ID.

Add libxc stubs, and extend xen-hptool with smt-{enable,disable} options.
These are intended to be shorthands for a loop over cpu-{online,offline}.

To simplify the implemention, they will strictly enable/disable secondary
siblings (those with a non-zero thread id).  This functionality is intended
for use in production scenarios where debugging options such as `maxcpus=` or
other manual plug/unplug configuration has not been used.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
Acked-by: Wei Liu <wei.liu2@citrix.com>

diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index a3628e56bb7bdda7eb32026b7cf36ec114d7e4f1..49a6b2ab05bd1b216a50ae3b73087f2a1261ba4f 100644 (file)
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -1854,6 +1854,8 @@ int xc_pm_reset_cxstat(xc_interface *xch, int cpuid);
 
 int xc_cpu_online(xc_interface *xch, int cpu);
 int xc_cpu_offline(xc_interface *xch, int cpu);
+int xc_smt_enable(xc_interface *xch);
+int xc_smt_disable(xc_interface *xch);
 
 /* 
  * cpufreq para name of this structure named 

diff --git a/tools/libxc/xc_cpu_hotplug.c b/tools/libxc/xc_cpu_hotplug.c
index 58c2a0fad6fbcc5f850306f9cd18b73a88b0ab47..2ea98257792ba0b10330de492e96c4b73860d7a8 100644 (file)
--- a/tools/libxc/xc_cpu_hotplug.c
+++ b/tools/libxc/xc_cpu_hotplug.c
@@ -46,3 +46,29 @@ int xc_cpu_offline(xc_interface *xch, int cpu)
     return ret;
 }
 
+int xc_smt_enable(xc_interface *xch)
+{
+    DECLARE_SYSCTL;
+    int ret;
+
+    sysctl.cmd = XEN_SYSCTL_cpu_hotplug;
+    sysctl.u.cpu_hotplug.cpu = 0;
+    sysctl.u.cpu_hotplug.op = XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE;
+    ret = xc_sysctl(xch, &sysctl);
+
+    return ret;
+}
+
+int xc_smt_disable(xc_interface *xch)
+{
+    DECLARE_SYSCTL;
+    int ret;
+
+    sysctl.cmd = XEN_SYSCTL_cpu_hotplug;
+    sysctl.u.cpu_hotplug.cpu = 0;
+    sysctl.u.cpu_hotplug.op = XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE;
+    ret = xc_sysctl(xch, &sysctl);
+
+    return ret;
+}
+

diff --git a/tools/misc/xen-hptool.c b/tools/misc/xen-hptool.c
index 40cd9667685a5f6e96c0ca253bc2fcf2757e77a6..6e27d9cf43d4a685865f353c60d2ad0a901f9525 100644 (file)
--- a/tools/misc/xen-hptool.c
+++ b/tools/misc/xen-hptool.c
@@ -19,6 +19,8 @@ void show_help(void)
             "  mem-online    <mfn>      online MEMORY <mfn>\n"
             "  mem-offline   <mfn>      offline MEMORY <mfn>\n"
             "  mem-status    <mfn>      query Memory status<mfn>\n"
+            "  smt-enable               onlines all SMT threads\n"
+            "  smt-disable              offlines all SMT threads\n"
            );
 }
 
@@ -304,6 +306,58 @@ static int hp_cpu_offline_func(int argc, char *argv[])
     return ret;
 }
 
+static int main_smt_enable(int argc, char *argv[])
+{
+    int ret;
+
+    if ( argc )
+    {
+        show_help();
+        return -1;
+    }
+
+    for ( ;; )
+    {
+        ret = xc_smt_enable(xch);
+        if ( (ret >= 0) || (errno != EBUSY) )
+            break;
+    }
+
+    if ( ret < 0 )
+        fprintf(stderr, "Unable to enable SMT: errno %d, %s\n",
+                errno, strerror(errno));
+    else
+        printf("Enabled SMT\n");
+
+    return ret;
+}
+
+static int main_smt_disable(int argc, char *argv[])
+{
+    int ret;
+
+    if ( argc )
+    {
+        show_help();
+        return -1;
+    }
+
+    for ( ;; )
+    {
+        ret = xc_smt_disable(xch);
+        if ( (ret >= 0) || (errno != EBUSY) )
+            break;
+    }
+
+    if ( ret < 0 )
+        fprintf(stderr, "Unable to disable SMT: errno %d, %s\n",
+                errno, strerror(errno));
+    else
+        printf("Disabled SMT\n");
+
+    return ret;
+}
+
 struct {
     const char *name;
     int (*function)(int argc, char *argv[]);
@@ -314,6 +368,8 @@ struct {
     { "mem-status", hp_mem_query_func},
     { "mem-online", hp_mem_online_func},
     { "mem-offline", hp_mem_offline_func},
+    { "smt-enable", main_smt_enable },
+    { "smt-disable", main_smt_disable },
 };
 
 

diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
index cff4415dedbab01376813bd6dbe3e907a1957273..3f06fecbd888460225dbcb926755bfbcdb06f5b6 100644 (file)
--- a/xen/arch/x86/sysctl.c
+++ b/xen/arch/x86/sysctl.c
@@ -114,6 +114,48 @@ long cpu_down_helper(void *data)
     return ret;
 }
 
+static long smt_up_down_helper(void *data)
+{
+    bool up = (bool)data;
+    unsigned int cpu, sibling_mask = boot_cpu_data.x86_num_siblings - 1;
+    int ret = 0;
+
+    opt_smt = up;
+
+    for_each_present_cpu ( cpu )
+    {
+        /* Skip primary siblings (those whose thread id is 0). */
+        if ( !(x86_cpu_to_apicid[cpu] & sibling_mask) )
+            continue;
+
+        ret = up ? cpu_up_helper(_p(cpu))
+                 : cpu_down_helper(_p(cpu));
+
+        if ( ret && ret != -EEXIST )
+            break;
+
+        /*
+         * Ensure forward progress by only considering preemption when we have
+         * changed the state of one or more cpus.
+         */
+        if ( ret != -EEXIST && general_preempt_check() )
+        {
+            /* In tasklet context - can't create a contination. */
+            ret = -EBUSY;
+            break;
+        }
+
+        ret = 0; /* Avoid exiting with -EEXIST in the success case. */
+    }
+
+    if ( !ret )
+        printk(XENLOG_INFO "SMT %s - online CPUs 0x%*pb\n",
+               up ? "enabled" : "disabled",
+               nr_cpu_ids, cpumask_bits(&cpu_online_map));
+
+    return ret;
+}
+
 void arch_do_physinfo(struct xen_sysctl_physinfo *pi)
 {
     memcpy(pi->hw_cap, boot_cpu_data.x86_capability,
@@ -137,11 +179,12 @@ long arch_do_sysctl(
     case XEN_SYSCTL_cpu_hotplug:
     {
         unsigned int cpu = sysctl->u.cpu_hotplug.cpu;
+        unsigned int op  = sysctl->u.cpu_hotplug.op;
         bool plug;
         long (*fn)(void *);
         void *hcpu;
 
-        switch ( sysctl->u.cpu_hotplug.op )
+        switch ( op )
         {
         case XEN_SYSCTL_CPU_HOTPLUG_ONLINE:
             plug = true;
@@ -155,6 +198,18 @@ long arch_do_sysctl(
             hcpu = _p(cpu);
             break;
 
+        case XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE:
+        case XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE:
+            if ( !cpu_has_htt || boot_cpu_data.x86_num_siblings < 2 )
+            {
+                ret = -EOPNOTSUPP;
+                break;
+            }
+            plug = op == XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE;
+            fn = smt_up_down_helper;
+            hcpu = _p(plug);
+            break;
+
         default:
             ret = -EOPNOTSUPP;
             break;

diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h
index c49b4dcc996413cec66bb39e9a0d5b0edbfee948..943be3d5ab55752d159f0ec80cdd8d0179941fa0 100644 (file)
--- a/xen/include/public/sysctl.h
+++ b/xen/include/public/sysctl.h
@@ -246,8 +246,27 @@ struct xen_sysctl_get_pmstat {
 struct xen_sysctl_cpu_hotplug {
     /* IN variables */
     uint32_t cpu;   /* Physical cpu. */
+
+    /* Single CPU enable/disable. */
 #define XEN_SYSCTL_CPU_HOTPLUG_ONLINE  0
 #define XEN_SYSCTL_CPU_HOTPLUG_OFFLINE 1
+
+    /*
+     * SMT enable/disable.
+     *
+     * These two ops loop over all present CPUs, and either online or offline
+     * every non-primary sibling thread (those with a thread id which is not
+     * 0).  This behaviour is chosen to simplify the implementation.
+     *
+     * They are intended as a shorthand for identifying and feeding the cpu
+     * numbers individually to HOTPLUG_{ON,OFF}LINE.
+     *
+     * These are not expected to be used in conjunction with debugging options
+     * such as `maxcpus=` or when other manual configuration of offline cpus
+     * is in use.
+     */
+#define XEN_SYSCTL_CPU_HOTPLUG_SMT_ENABLE  2
+#define XEN_SYSCTL_CPU_HOTPLUG_SMT_DISABLE 3
     uint32_t op;    /* hotplug opcode */
 };