From: Michal Privoznik <mprivozn@redhat.com>
Date: Wed, 27 May 2020 11:42:22 +0000 (+0200)
Subject: Allow NUMA nodes without vCPUs
X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=a26f61ee0cffa421b87ef568002b684dd8025432;p=libvirt.git

Allow NUMA nodes without vCPUs

QEMU allows creating NUMA nodes that have memory only.
These are somehow important for HMAT.

With check done in qemuValidateDomainDef() for QEMU 2.7 or newer
(checked via QEMU_CAPS_NUMA), we can be sure that the vCPUs are
fully assigned to NUMA nodes in domain XML.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Daniel Henrique Barboza <danielhb413@gmail.com>
---

diff --git a/docs/formatdomain.html.in b/docs/formatdomain.html.in
index 562bd3e538..d3f753c8d1 100644
--- a/docs/formatdomain.html.in
+++ b/docs/formatdomain.html.in
@@ -1840,6 +1840,8 @@
       consistent across qemu and libvirt versions.
       <code>memory</code> specifies the node memory
       in kibibytes (i.e. blocks of 1024 bytes).
+      <span class="since">Since 6.6.0</span> the <code>cpus</code> attribute
+      is optional and if omitted a CPU-less NUMA node is created.
       <span class="since">Since 1.2.11</span> one can use an additional <a
           href="#elementsMemoryAllocation"><code>unit</code></a> attribute to
       define units in which <code>memory</code> is specified.
diff --git a/docs/schemas/cputypes.rng b/docs/schemas/cputypes.rng
index e2744acad3..a1682a1003 100644
--- a/docs/schemas/cputypes.rng
+++ b/docs/schemas/cputypes.rng
@@ -115,9 +115,11 @@
           <ref name="unsignedInt"/>
         </attribute>
       </optional>
-      <attribute name="cpus">
-        <ref name="cpuset"/>
-      </attribute>
+      <optional>
+        <attribute name="cpus">
+          <ref name="cpuset"/>
+        </attribute>
+      </optional>
       <attribute name="memory">
         <ref name="memoryKB"/>
       </attribute>
diff --git a/src/conf/numa_conf.c b/src/conf/numa_conf.c
index 7b9084456e..7cf62ce7da 100644
--- a/src/conf/numa_conf.c
+++ b/src/conf/numa_conf.c
@@ -889,32 +889,28 @@ virDomainNumaDefParseXML(virDomainNumaPtr def,
         }
         VIR_FREE(tmp);
 
-        if (def->mem_nodes[cur_cell].cpumask) {
+        if (def->mem_nodes[cur_cell].mem) {
             virReportError(VIR_ERR_XML_ERROR,
                            _("Duplicate NUMA cell info for cell id '%u'"),
                            cur_cell);
             goto cleanup;
         }
 
-        if (!(tmp = virXMLPropString(nodes[i], "cpus"))) {
-            virReportError(VIR_ERR_XML_ERROR, "%s",
-                           _("Missing 'cpus' attribute in NUMA cell"));
-            goto cleanup;
-        }
+        if ((tmp = virXMLPropString(nodes[i], "cpus"))) {
+            g_autoptr(virBitmap) cpumask = NULL;
 
-        if (virBitmapParse(tmp, &def->mem_nodes[cur_cell].cpumask,
-                           VIR_DOMAIN_CPUMASK_LEN) < 0)
-            goto cleanup;
+            if (virBitmapParse(tmp, &cpumask, VIR_DOMAIN_CPUMASK_LEN) < 0)
+                goto cleanup;
 
-        if (virBitmapIsAllClear(def->mem_nodes[cur_cell].cpumask)) {
-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-                          _("NUMA cell %d has no vCPUs assigned"), cur_cell);
-            goto cleanup;
+            if (!virBitmapIsAllClear(cpumask))
+                def->mem_nodes[cur_cell].cpumask = g_steal_pointer(&cpumask);
+            VIR_FREE(tmp);
         }
-        VIR_FREE(tmp);
 
         for (j = 0; j < n; j++) {
-            if (j == cur_cell || !def->mem_nodes[j].cpumask)
+            if (j == cur_cell ||
+                !def->mem_nodes[j].cpumask ||
+                !def->mem_nodes[cur_cell].cpumask)
                 continue;
 
             if (virBitmapOverlaps(def->mem_nodes[j].cpumask,
@@ -975,7 +971,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
 {
     virDomainMemoryAccess memAccess;
     virTristateBool discard;
-    char *cpustr;
     size_t ncells = virDomainNumaGetNodeCount(def);
     size_t i;
 
@@ -985,17 +980,22 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
     virBufferAddLit(buf, "<numa>\n");
     virBufferAdjustIndent(buf, 2);
     for (i = 0; i < ncells; i++) {
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def, i);
         int ndistances;
 
         memAccess = virDomainNumaGetNodeMemoryAccessMode(def, i);
         discard = virDomainNumaGetNodeDiscard(def, i);
 
-        if (!(cpustr = virBitmapFormat(virDomainNumaGetNodeCpumask(def, i))))
-            return -1;
-
         virBufferAddLit(buf, "<cell");
         virBufferAsprintf(buf, " id='%zu'", i);
-        virBufferAsprintf(buf, " cpus='%s'", cpustr);
+
+        if (cpumask) {
+            g_autofree char *cpustr = virBitmapFormat(cpumask);
+
+            if (!cpustr)
+                return -1;
+            virBufferAsprintf(buf, " cpus='%s'", cpustr);
+        }
         virBufferAsprintf(buf, " memory='%llu'",
                           virDomainNumaGetNodeMemorySize(def, i));
         virBufferAddLit(buf, " unit='KiB'");
@@ -1031,8 +1031,6 @@ virDomainNumaDefFormatXML(virBufferPtr buf,
             virBufferAdjustIndent(buf, -2);
             virBufferAddLit(buf, "</cell>\n");
         }
-
-        VIR_FREE(cpustr);
     }
     virBufferAdjustIndent(buf, -2);
     virBufferAddLit(buf, "</numa>\n");
@@ -1047,8 +1045,12 @@ virDomainNumaGetCPUCountTotal(virDomainNumaPtr numa)
     size_t i;
     unsigned int ret = 0;
 
-    for (i = 0; i < numa->nmem_nodes; i++)
-        ret += virBitmapCountBits(virDomainNumaGetNodeCpumask(numa, i));
+    for (i = 0; i < numa->nmem_nodes; i++) {
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i);
+
+        if (cpumask)
+            ret += virBitmapCountBits(cpumask);
+    }
 
     return ret;
 }
@@ -1060,11 +1062,14 @@ virDomainNumaGetMaxCPUID(virDomainNumaPtr numa)
     unsigned int ret = 0;
 
     for (i = 0; i < numa->nmem_nodes; i++) {
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, i);
         int bit;
 
-        bit = virBitmapLastSetBit(virDomainNumaGetNodeCpumask(numa, i));
-        if (bit > ret)
-            ret = bit;
+        if (cpumask) {
+            bit = virBitmapLastSetBit(cpumask);
+            if (bit > ret)
+                ret = bit;
+        }
     }
 
     return ret;
diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
index 1c1279e1cb..b99ddd150b 100644
--- a/src/libxl/xen_xl.c
+++ b/src/libxl/xen_xl.c
@@ -1444,19 +1444,21 @@ xenFormatXLVnuma(virConfValuePtr list,
 {
     int ret = -1;
     size_t i;
-
     virBuffer buf = VIR_BUFFER_INITIALIZER;
     virConfValuePtr numaVnode, tmp;
-
+    virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(numa, node);
     size_t nodeSize = virDomainNumaGetNodeMemorySize(numa, node) / 1024;
-    char *nodeVcpus = virBitmapFormat(virDomainNumaGetNodeCpumask(numa, node));
+    g_autofree char *nodeVcpus = NULL;
 
-    if (VIR_ALLOC(numaVnode) < 0)
+    if (!cpumask ||
+        VIR_ALLOC(numaVnode) < 0)
         goto cleanup;
 
     numaVnode->type = VIR_CONF_LIST;
     numaVnode->list = NULL;
 
+    nodeVcpus = virBitmapFormat(cpumask);
+
     /* pnode */
     virBufferAsprintf(&buf, "pnode=%zu", node);
     xenFormatXLVnode(numaVnode, &buf);
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 2f0f1afe4c..b7931c75e9 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -7124,8 +7124,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
     size_t i, j;
     virQEMUCapsPtr qemuCaps = priv->qemuCaps;
     g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
-    char *cpumask = NULL;
-    char *tmpmask = NULL;
     char *next = NULL;
     virBufferPtr nodeBackends = NULL;
     bool needBackend = false;
@@ -7170,9 +7168,7 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
         goto cleanup;
 
     for (i = 0; i < ncells; i++) {
-        VIR_FREE(cpumask);
-        if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i))))
-            goto cleanup;
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);
 
         if (needBackend) {
             virCommandAddArg(cmd, "-object");
@@ -7182,11 +7178,19 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
         virCommandAddArg(cmd, "-numa");
         virBufferAsprintf(&buf, "node,nodeid=%zu", i);
 
-        for (tmpmask = cpumask; tmpmask; tmpmask = next) {
-            if ((next = strchr(tmpmask, ',')))
-                *(next++) = '\0';
-            virBufferAddLit(&buf, ",cpus=");
-            virBufferAdd(&buf, tmpmask, -1);
+        if (cpumask) {
+            g_autofree char *cpumaskStr = NULL;
+            char *tmpmask;
+
+            if (!(cpumaskStr = virBitmapFormat(cpumask)))
+                goto cleanup;
+
+            for (tmpmask = cpumaskStr; tmpmask; tmpmask = next) {
+                if ((next = strchr(tmpmask, ',')))
+                    *(next++) = '\0';
+                virBufferAddLit(&buf, ",cpus=");
+                virBufferAdd(&buf, tmpmask, -1);
+            }
         }
 
         if (needBackend)
@@ -7217,8 +7221,6 @@ qemuBuildNumaCommandLine(virQEMUDriverConfigPtr cfg,
     ret = 0;
 
  cleanup:
-    VIR_FREE(cpumask);
-
     if (nodeBackends) {
         for (i = 0; i < ncells; i++)
             virBufferFreeAndReset(&nodeBackends[i]);
diff --git a/src/qemu/qemu_validate.c b/src/qemu/qemu_validate.c
index 6d0ffd2013..f0072ceec5 100644
--- a/src/qemu/qemu_validate.c
+++ b/src/qemu/qemu_validate.c
@@ -654,7 +654,7 @@ qemuValidateDomainDefNuma(const virDomainDef *def,
     }
 
     for (i = 0; i < ncells; i++) {
-        g_autofree char * cpumask = NULL;
+        virBitmapPtr cpumask = virDomainNumaGetNodeCpumask(def->numa, i);
 
         if (!hasMemoryCap &&
             virDomainNumaGetNodeMemoryAccessMode(def->numa, i)) {
@@ -664,17 +664,19 @@ qemuValidateDomainDefNuma(const virDomainDef *def,
             return -1;
         }
 
-        if (!(cpumask = virBitmapFormat(virDomainNumaGetNodeCpumask(def->numa, i))))
-            return -1;
+        if (cpumask) {
+            g_autofree char * cpumaskStr = NULL;
+            if (!(cpumaskStr = virBitmapFormat(cpumask)))
+                return -1;
 
-        if (strchr(cpumask, ',') &&
-            !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) {
-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                           _("disjoint NUMA cpu ranges are not supported "
-                             "with this QEMU"));
-            return -1;
+            if (strchr(cpumaskStr, ',') &&
+                !virQEMUCapsGet(qemuCaps, QEMU_CAPS_NUMA)) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                               _("disjoint NUMA cpu ranges are not supported "
+                                 "with this QEMU"));
+                return -1;
+            }
         }
-
     }
 
     if (virDomainNumaNodesDistancesAreBeingSet(def->numa) &&
diff --git a/tests/qemuxml2argvdata/numatune-no-vcpu.args b/tests/qemuxml2argvdata/numatune-no-vcpu.args
new file mode 100644
index 0000000000..a1f1ee044e
--- /dev/null
+++ b/tests/qemuxml2argvdata/numatune-no-vcpu.args
@@ -0,0 +1,33 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/tmp/lib/domain--1-QEMUGuest \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest/.local/share \
+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest/.cache \
+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest/.config \
+QEMU_AUDIO_DRV=none \
+/usr/bin/qemu-system-x86_64 \
+-name QEMUGuest \
+-S \
+-machine pc,accel=tcg,usb=off,dump-guest-core=off \
+-m 12288 \
+-realtime mlock=off \
+-smp 12,sockets=12,cores=1,threads=1 \
+-numa node,nodeid=0,cpus=0-3,mem=2048 \
+-numa node,nodeid=1,cpus=4-7,mem=2048 \
+-numa node,nodeid=2,cpus=8-11,mem=2048 \
+-numa node,nodeid=3,mem=2048 \
+-numa node,nodeid=4,mem=2048 \
+-numa node,nodeid=5,mem=2048 \
+-uuid c7a5fdb2-cdaf-9455-926a-d65c16db1809 \
+-display none \
+-no-user-config \
+-nodefaults \
+-chardev socket,id=charmonitor,path=/tmp/lib/domain--1-QEMUGuest/monitor.sock,\
+server,nowait \
+-mon chardev=charmonitor,id=monitor,mode=control \
+-rtc base=utc \
+-no-shutdown \
+-usb \
+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3
diff --git a/tests/qemuxml2argvdata/numatune-no-vcpu.xml b/tests/qemuxml2argvdata/numatune-no-vcpu.xml
new file mode 100644
index 0000000000..f25a07d7ed
--- /dev/null
+++ b/tests/qemuxml2argvdata/numatune-no-vcpu.xml
@@ -0,0 +1,42 @@
+<domain type='qemu'>
+  <name>QEMUGuest</name>
+  <uuid>c7a5fdb2-cdaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>12582912</memory>
+  <currentMemory unit='KiB'>12582912</currentMemory>
+  <vcpu placement='static'>12</vcpu>
+  <os>
+    <type arch='x86_64' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <cpu>
+    <numa>
+      <cell id='0' cpus='0-3' memory='2097152' unit='KiB'/>
+      <cell id='1' cpus='4-7' memory='2097152' unit='KiB'/>
+      <cell id='2' cpus='8-11' memory='2097152' unit='KiB'/>
+      <cell id='3' memory='2097152' unit='KiB'/>
+      <cell id='4' memory='2097152' unit='KiB'/>
+      <cell id='5' memory='2097152' unit='KiB'/>
+    </numa>
+  </cpu>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>restart</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu-system-x86_64</emulator>
+    <controller type='usb' index='0'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+    </controller>
+    <controller type='pci' index='0' model='pci-root'/>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <memballoon model='virtio'>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </memballoon>
+  </devices>
+</domain>
diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
index 26333d8f40..b68cbe1d80 100644
--- a/tests/qemuxml2argvtest.c
+++ b/tests/qemuxml2argvtest.c
@@ -1945,6 +1945,7 @@ mymain(void)
     DO_TEST_PARSE_ERROR("numatune-memnode-no-memory", NONE);
 
     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
+    DO_TEST("numatune-no-vcpu", NONE);
 
     DO_TEST("numatune-auto-nodeset-invalid", NONE);
     DO_TEST("numatune-auto-prefer", QEMU_CAPS_OBJECT_MEMORY_RAM,
diff --git a/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml b/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml
new file mode 120000
index 0000000000..f213032685
--- /dev/null
+++ b/tests/qemuxml2xmloutdata/numatune-no-vcpu.xml
@@ -0,0 +1 @@
+../qemuxml2argvdata/numatune-no-vcpu.xml
\ No newline at end of file
diff --git a/tests/qemuxml2xmltest.c b/tests/qemuxml2xmltest.c
index c77619075f..6a604e2a05 100644
--- a/tests/qemuxml2xmltest.c
+++ b/tests/qemuxml2xmltest.c
@@ -1126,6 +1126,7 @@ mymain(void)
     DO_TEST("numatune-memnode", QEMU_CAPS_NUMA, QEMU_CAPS_OBJECT_MEMORY_FILE);
     DO_TEST("numatune-memnode-no-memory", QEMU_CAPS_OBJECT_MEMORY_FILE);
     DO_TEST("numatune-distances", QEMU_CAPS_NUMA, QEMU_CAPS_NUMA_DIST);
+    DO_TEST("numatune-no-vcpu", QEMU_CAPS_NUMA);
 
     DO_TEST("bios-nvram", NONE);
     DO_TEST("bios-nvram-os-interleave", NONE);