libxl: set vcpu affinity during domain creation

Since Xen 4.5 libxl allows to set affinities during domain creation.
This enables Xen to allocate the domain memory on NUMA systems close to
the specified pcpus.

Libvirt can now handle <domain/cputune/vcpupin> in domU.xml correctly.

Without this change, Xen will create the domU and assign NUMA memory and
vcpu affinities on its own. Later libvirt will adjust the affinity,
which may move the vcpus away from the assigned NUMA node.

Signed-off-by: Olaf Hering <olaf@aepfle.de>
Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>
Reviewed-by: Jim Fehlig <jfehlig@suse.com>

diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
index e33297a9bad6566428dd84b37e5f661a80099296..5623bdf6fb13bb3008a5cd18c89ab7346d51e248 100644 (file)
--- a/src/libxl/libxl_conf.c
+++ b/src/libxl/libxl_conf.c
@@ -284,6 +284,56 @@ libxlMakeChrdevStr(virDomainChrDef *def, char **buf)
     return 0;
 }
 
+static int
+libxlSetVcpuAffinities(virDomainDef *def,
+                       libxl_ctx *ctx,
+                       libxl_domain_build_info *b_info)
+{
+    libxl_bitmap *vcpu_affinity_array;
+    unsigned int vcpuid;
+    unsigned int vcpu_idx = 0;
+    virDomainVcpuDef *vcpu;
+    bool has_vcpu_pin = false;
+
+    /* Get highest vcpuid with cpumask */
+    for (vcpuid = 0; vcpuid < b_info->max_vcpus; vcpuid++) {
+        vcpu = virDomainDefGetVcpu(def, vcpuid);
+        if (!vcpu)
+            continue;
+        if (!vcpu->cpumask)
+            continue;
+        vcpu_idx = vcpuid;
+        has_vcpu_pin = true;
+    }
+    /* Nothing to do */
+    if (!has_vcpu_pin)
+        return 0;
+
+    /* Adjust index */
+    vcpu_idx++;
+
+    b_info->num_vcpu_hard_affinity = vcpu_idx;
+    /* Will be released by libxl_domain_config_dispose */
+    b_info->vcpu_hard_affinity = g_new0(libxl_bitmap, vcpu_idx);
+    vcpu_affinity_array = b_info->vcpu_hard_affinity;
+
+    for (vcpuid = 0; vcpuid < vcpu_idx; vcpuid++) {
+        libxl_bitmap *map = &vcpu_affinity_array[vcpuid];
+        libxl_bitmap_init(map);
+        /* libxl owns the bitmap */
+        if (libxl_cpu_bitmap_alloc(ctx, map, 0))
+            return -1;
+        vcpu = virDomainDefGetVcpu(def, vcpuid);
+        /* Apply the given mask, or allow unhandled vcpus to run anywhere */
+        if (vcpu && vcpu->cpumask)
+            virBitmapToDataBuf(vcpu->cpumask, map->map, map->size);
+        else
+            libxl_bitmap_set_any(map);
+    }
+    libxl_defbool_set(&b_info->numa_placement, false);
+    return 0;
+}
+
 static int
 libxlMakeDomBuildInfo(virDomainDef *def,
                       libxlDriverConfig *cfg,
@@ -321,6 +371,9 @@ libxlMakeDomBuildInfo(virDomainDef *def,
     for (i = 0; i < virDomainDefGetVcpus(def); i++)
         libxl_bitmap_set((&b_info->avail_vcpus), i);
 
+    if (libxlSetVcpuAffinities(def, ctx, b_info))
+        return -1;
+
     switch ((virDomainClockOffsetType) clock.offset) {
     case VIR_DOMAIN_CLOCK_OFFSET_VARIABLE:
         if (clock.data.variable.basis == VIR_DOMAIN_CLOCK_BASIS_LOCALTIME)

diff --git a/src/libxl/libxl_domain.c b/src/libxl/libxl_domain.c
index d78765ad840e748038f353d1d906e061450a63e4..625e04a9b0ba0112dac5b5177cb3a1d713116adf 100644 (file)
--- a/src/libxl/libxl_domain.c
+++ b/src/libxl/libxl_domain.c
@@ -967,49 +967,6 @@ libxlDomainAutoCoreDump(libxlDriverPrivate *driver,
     return 0;
 }
 
-int
-libxlDomainSetVcpuAffinities(libxlDriverPrivate *driver, virDomainObj *vm)
-{
-    g_autoptr(libxlDriverConfig) cfg = libxlDriverConfigGet(driver);
-    virDomainVcpuDef *vcpu;
-    libxl_bitmap map;
-    virBitmap *cpumask = NULL;
-    size_t i;
-    int ret = -1;
-
-    libxl_bitmap_init(&map);
-
-    for (i = 0; i < virDomainDefGetVcpus(vm->def); ++i) {
-        vcpu = virDomainDefGetVcpu(vm->def, i);
-
-        if (!vcpu->online)
-            continue;
-
-        if (!(cpumask = vcpu->cpumask))
-            cpumask = vm->def->cpumask;
-
-        if (!cpumask)
-            continue;
-
-        if (virBitmapToData(cpumask, &map.map, (int *)&map.size) < 0)
-            goto cleanup;
-
-        if (libxl_set_vcpuaffinity(cfg->ctx, vm->def->id, i, &map, NULL) != 0) {
-            virReportError(VIR_ERR_INTERNAL_ERROR,
-                           _("Failed to pin vcpu '%zu' with libxenlight"), i);
-            goto cleanup;
-        }
-
-        libxl_bitmap_dispose(&map); /* Also returns to freshly-init'd state */
-    }
-
-    ret = 0;
-
- cleanup:
-    libxl_bitmap_dispose(&map);
-    return ret;
-}
-
 static int
 libxlDomainFreeMem(libxl_ctx *ctx, libxl_domain_config *d_config)
 {
@@ -1460,9 +1417,6 @@ libxlDomainStart(libxlDriverPrivate *driver,
         goto destroy_dom;
     }
 
-    if (libxlDomainSetVcpuAffinities(driver, vm) < 0)
-        goto destroy_dom;
-
     if (!start_paused) {
         libxlDomainUnpauseWrapper(cfg->ctx, domid);
         virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, VIR_DOMAIN_RUNNING_BOOTED);

diff --git a/src/libxl/libxl_domain.h b/src/libxl/libxl_domain.h
index cbe7ba19baf4aebb405ea27b0178d7a6fffec8a4..928ee8f5cd800090c6416f814b823d4d423d92bf 100644 (file)
--- a/src/libxl/libxl_domain.h
+++ b/src/libxl/libxl_domain.h
@@ -124,10 +124,6 @@ int
 libxlDomainAutoCoreDump(libxlDriverPrivate *driver,
                         virDomainObj *vm);
 
-int
-libxlDomainSetVcpuAffinities(libxlDriverPrivate *driver,
-                             virDomainObj *vm);
-
 int
 libxlDomainStartNew(libxlDriverPrivate *driver,
                     virDomainObj *vm,