]> xenbits.xensource.com Git - libvirt.git/commitdiff
qemu: Create hugepage path on per domain basis
authorMichal Privoznik <mprivozn@redhat.com>
Tue, 22 Nov 2016 12:21:51 +0000 (13:21 +0100)
committerMichal Privoznik <mprivozn@redhat.com>
Thu, 8 Dec 2016 14:45:52 +0000 (15:45 +0100)
If you've ever tried running a huge page backed guest under
different user than in qemu.conf, you probably failed. Problem is
even though we have corresponding APIs in the security drivers,
there's no implementation and thus we don't relabel the huge page
path. But even if we did, so far all of the domains share the
same path:

   /hugepageMount/libvirt/qemu

Our only option there would be to set 0777 mode on the qemu dir
which is totally unsafe. Therefore, we can create dir on
per-domain basis, i.e.:

   /hugepageMount/libvirt/qemu/domainName

and chown domainName dir to the user that domain is configured to
run under.

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
14 files changed:
src/qemu/qemu_command.c
src/qemu/qemu_conf.c
src/qemu/qemu_conf.h
src/qemu/qemu_driver.c
src/qemu/qemu_process.c
tests/qemuxml2argvdata/qemuxml2argv-hugepages-numa.args
tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages.args
tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages2.args
tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages3.args
tests/qemuxml2argvdata/qemuxml2argv-hugepages-pages5.args
tests/qemuxml2argvdata/qemuxml2argv-hugepages-shared.args
tests/qemuxml2argvdata/qemuxml2argv-hugepages.args
tests/qemuxml2argvdata/qemuxml2argv-memory-hotplug-dimm-addr.args
tests/qemuxml2argvdata/qemuxml2argv-memory-hotplug-dimm.args

index 90a9c5544dd623eaa33b35e1165805d1648889f9..f8e48d20899d3484e043c9f585498797c09d011e 100644 (file)
@@ -3316,7 +3316,7 @@ qemuBuildMemoryBackendStr(unsigned long long size,
         return -1;
 
     if (pagesize) {
-        if (qemuGetHupageMemPath(cfg, pagesize, &mem_path) < 0)
+        if (qemuGetDomainHupageMemPath(def, cfg, pagesize, &mem_path) < 0)
             goto cleanup;
 
         *backendType = "memory-backend-file";
@@ -7270,7 +7270,7 @@ qemuBuildMemPathStr(virQEMUDriverConfigPtr cfg,
         return -1;
     }
 
-    if (qemuGetHupageMemPath(cfg, def->mem.hugepages[0].size, &mem_path) < 0)
+    if (qemuGetDomainHupageMemPath(def, cfg, def->mem.hugepages[0].size, &mem_path) < 0)
         return -1;
 
     virCommandAddArgList(cmd, "-mem-prealloc", "-mem-path", mem_path, NULL);
index ccefbe89077c665e67e3a33e11cc9f8de5d451c5..ad85551e9cb2bb33c6c42337feeebc6e58bd9c54 100644 (file)
@@ -1456,7 +1456,7 @@ qemuTranslateSnapshotDiskSourcePool(virConnectPtr conn ATTRIBUTE_UNUSED,
 }
 
 char *
-qemuGetHugepagePath(virHugeTLBFSPtr hugepage)
+qemuGetBaseHugepagePath(virHugeTLBFSPtr hugepage)
 {
     char *ret;
 
@@ -1467,8 +1467,25 @@ qemuGetHugepagePath(virHugeTLBFSPtr hugepage)
 }
 
 
+char *
+qemuGetDomainHugepagePath(const virDomainDef *def,
+                          virHugeTLBFSPtr hugepage)
+{
+    char *base = qemuGetBaseHugepagePath(hugepage);
+    char *domPath = virDomainObjGetShortName(def);
+    char *ret;
+
+    if (base && domPath)
+        ignore_value(virAsprintf(&ret, "%s/%s", base, domPath));
+    VIR_FREE(domPath);
+    VIR_FREE(base);
+    return ret;
+}
+
+
 /**
- * qemuGetDefaultHugepath:
+ * qemuGetDomainDefaultHugepath:
+ * @def: domain definition
  * @hugetlbfs: array of configured hugepages
  * @nhugetlbfs: number of item in the array
  *
@@ -1477,8 +1494,9 @@ qemuGetHugepagePath(virHugeTLBFSPtr hugepage)
  * Returns 0 on success, -1 otherwise.
  * */
 char *
-qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
-                       size_t nhugetlbfs)
+qemuGetDomainDefaultHugepath(const virDomainDef *def,
+                             virHugeTLBFSPtr hugetlbfs,
+                             size_t nhugetlbfs)
 {
     size_t i;
 
@@ -1489,12 +1507,12 @@ qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
     if (i == nhugetlbfs)
         i = 0;
 
-    return qemuGetHugepagePath(&hugetlbfs[i]);
+    return qemuGetDomainHugepagePath(def, &hugetlbfs[i]);
 }
 
 
 /**
- * qemuGetHupageMemPath: Construct HP enabled memory backend path
+ * qemuGetDomainHupageMemPath: Construct HP enabled memory backend path
  *
  * If no specific hugepage size is requested (@pagesize is zero)
  * the default hugepage size is used).
@@ -1504,9 +1522,10 @@ qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
  *        -1 otherwise.
  */
 int
-qemuGetHupageMemPath(virQEMUDriverConfigPtr cfg,
-                     unsigned long long pagesize,
-                     char **memPath)
+qemuGetDomainHupageMemPath(const virDomainDef *def,
+                           virQEMUDriverConfigPtr cfg,
+                           unsigned long long pagesize,
+                           char **memPath)
 {
     size_t i = 0;
 
@@ -1518,8 +1537,9 @@ qemuGetHupageMemPath(virQEMUDriverConfigPtr cfg,
     }
 
     if (!pagesize) {
-        if (!(*memPath = qemuGetDefaultHugepath(cfg->hugetlbfs,
-                                                cfg->nhugetlbfs)))
+        if (!(*memPath = qemuGetDomainDefaultHugepath(def,
+                                                      cfg->hugetlbfs,
+                                                      cfg->nhugetlbfs)))
             return -1;
     } else {
         for (i = 0; i < cfg->nhugetlbfs; i++) {
@@ -1535,7 +1555,7 @@ qemuGetHupageMemPath(virQEMUDriverConfigPtr cfg,
             return -1;
         }
 
-        if (!(*memPath = qemuGetHugepagePath(&cfg->hugetlbfs[i])))
+        if (!(*memPath = qemuGetDomainHugepagePath(def, &cfg->hugetlbfs[i])))
             return -1;
     }
 
index f6e325760488c939d3dde0ba8f7ff2bd58ccc337..d191e10565b90f5615e53add290f48e6d9bcea7c 100644 (file)
@@ -335,11 +335,15 @@ virDomainXMLOptionPtr virQEMUDriverCreateXMLConf(virQEMUDriverPtr driver);
 int qemuTranslateSnapshotDiskSourcePool(virConnectPtr conn,
                                         virDomainSnapshotDiskDefPtr def);
 
-char * qemuGetHugepagePath(virHugeTLBFSPtr hugepage);
-char * qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs,
-                              size_t nhugetlbfs);
-
-int qemuGetHupageMemPath(virQEMUDriverConfigPtr cfg,
-                         unsigned long long pagesize,
-                         char **memPath);
+char * qemuGetBaseHugepagePath(virHugeTLBFSPtr hugepage);
+char * qemuGetDomainHugepagePath(const virDomainDef *def,
+                                 virHugeTLBFSPtr hugepage);
+char * qemuGetDomainDefaultHugepath(const virDomainDef *def,
+                                    virHugeTLBFSPtr hugetlbfs,
+                                    size_t nhugetlbfs);
+
+int qemuGetDomainHupageMemPath(const virDomainDef *def,
+                               virQEMUDriverConfigPtr cfg,
+                               unsigned long long pagesize,
+                               char **memPath);
 #endif /* __QEMUD_CONF_H */
index 3f4e06bc3fdc501766b99a379832dac560fcc3d7..a8ae3fe0d009bcfb2a2db1828df9ab3f0a037527 100644 (file)
@@ -857,7 +857,7 @@ qemuStateInitialize(bool privileged,
      * it, since we can't assume the root mount point has permissions that
      * will let our spawned QEMU instances use it. */
     for (i = 0; i < cfg->nhugetlbfs; i++) {
-        hugepagePath = qemuGetHugepagePath(&cfg->hugetlbfs[i]);
+        hugepagePath = qemuGetBaseHugepagePath(&cfg->hugetlbfs[i]);
 
         if (!hugepagePath)
             goto error;
@@ -868,19 +868,10 @@ qemuStateInitialize(bool privileged,
                                  hugepagePath);
             goto error;
         }
-        if (privileged) {
-            if (virFileUpdatePerm(cfg->hugetlbfs[i].mnt_dir,
-                                  0, S_IXGRP | S_IXOTH) < 0)
-                goto error;
-            if (chown(hugepagePath, cfg->user, cfg->group) < 0) {
-                virReportSystemError(errno,
-                                     _("unable to set ownership on %s to %d:%d"),
-                                     hugepagePath,
-                                     (int) cfg->user,
-                                     (int) cfg->group);
-                goto error;
-            }
-        }
+        if (privileged &&
+            virFileUpdatePerm(cfg->hugetlbfs[i].mnt_dir,
+                              0, S_IXGRP | S_IXOTH) < 0)
+            goto error;
         VIR_FREE(hugepagePath);
     }
 
index 7f19c691e62980d197741ace3b0d0f431126f779..4241263a72fe410a6fc262450db6ec6acf4cb307 100644 (file)
@@ -3224,6 +3224,54 @@ qemuProcessReconnectCheckMemAliasOrderMismatch(virDomainObjPtr vm)
 }
 
 
+static int
+qemuProcessBuildDestroyHugepagesPath(virQEMUDriverPtr driver,
+                                     virDomainObjPtr vm,
+                                     bool build)
+{
+    virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
+    char *hugepagePath = NULL;
+    size_t i;
+    int ret = -1;
+
+    if (vm->def->mem.nhugepages) {
+        for (i = 0; i < cfg->nhugetlbfs; i++) {
+            VIR_FREE(hugepagePath);
+            hugepagePath = qemuGetDomainHugepagePath(vm->def, &cfg->hugetlbfs[i]);
+
+            if (!hugepagePath)
+                goto cleanup;
+
+            if (build) {
+                if (virFileMakePathWithMode(hugepagePath, 0700) < 0) {
+                    virReportSystemError(errno,
+                                         _("Unable to create %s"),
+                                         hugepagePath);
+                    goto cleanup;
+                }
+
+                if (virSecurityManagerSetHugepages(driver->securityManager,
+                                                   vm->def, hugepagePath) < 0) {
+                    virReportError(VIR_ERR_INTERNAL_ERROR,
+                                   "%s", _("Unable to set huge path in security driver"));
+                    goto cleanup;
+                }
+            } else {
+                if (rmdir(hugepagePath) < 0)
+                    VIR_WARN("Unable to remove hugepage path: %s (errno=%d)",
+                             hugepagePath, errno);
+            }
+        }
+    }
+
+    ret = 0;
+ cleanup:
+    VIR_FREE(hugepagePath);
+    virObjectUnref(cfg);
+    return ret;
+}
+
+
 struct qemuProcessReconnectData {
     virConnectPtr conn;
     virQEMUDriverPtr driver;
@@ -3368,6 +3416,9 @@ qemuProcessReconnect(void *opaque)
         goto cleanup;
     }
 
+    if (qemuProcessBuildDestroyHugepagesPath(driver, obj, true) < 0)
+        goto error;
+
     if ((qemuDomainAssignAddresses(obj->def, priv->qemuCaps,
                                    driver, obj, false)) < 0) {
         goto error;
@@ -5233,7 +5284,6 @@ qemuProcessPrepareHost(virQEMUDriverPtr driver,
 {
     int ret = -1;
     unsigned int hostdev_flags = 0;
-    size_t i;
     qemuDomainObjPrivatePtr priv = vm->privateData;
     virQEMUDriverConfigPtr cfg = virQEMUDriverGetConfig(driver);
 
@@ -5265,23 +5315,8 @@ qemuProcessPrepareHost(virQEMUDriverPtr driver,
                                NULL) < 0)
         goto cleanup;
 
-    if (vm->def->mem.nhugepages) {
-        for (i = 0; i < cfg->nhugetlbfs; i++) {
-            char *hugepagePath = qemuGetHugepagePath(&cfg->hugetlbfs[i]);
-
-            if (!hugepagePath)
-                goto cleanup;
-
-            if (virSecurityManagerSetHugepages(driver->securityManager,
-                                               vm->def, hugepagePath) < 0) {
-                virReportError(VIR_ERR_INTERNAL_ERROR,
-                               "%s", _("Unable to set huge path in security driver"));
-                VIR_FREE(hugepagePath);
-                goto cleanup;
-            }
-            VIR_FREE(hugepagePath);
-        }
-    }
+    if (qemuProcessBuildDestroyHugepagesPath(driver, vm, true) < 0)
+        goto cleanup;
 
     /* Ensure no historical cgroup for this VM is lying around bogus
      * settings */
@@ -5959,6 +5994,8 @@ void qemuProcessStop(virQEMUDriverPtr driver,
         goto endjob;
     }
 
+    qemuProcessBuildDestroyHugepagesPath(driver, vm, false);
+
     vm->def->id = -1;
 
     if (virAtomicIntDecAndTest(&driver->nactive) && driver->inhibitCallback)
index 7b90784020246d56d760f178c3f7cae5dfaab29c..26e9496cf2622401f62940c68c6e031f38b35374 100644 (file)
@@ -11,11 +11,11 @@ QEMU_AUDIO_DRV=spice \
 -m size=1048576k,slots=16,maxmem=1099511627776k \
 -smp 2,sockets=2,cores=1,threads=1 \
 -mem-prealloc \
--mem-path /dev/hugepages2M/libvirt/qemu \
+-mem-path /dev/hugepages2M/libvirt/qemu/-1-fedora \
 -numa node,nodeid=0,cpus=0-1,mem=1024 \
 -object memory-backend-file,id=memdimm0,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=1-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-fedora,size=1073741824,\
+host-nodes=1-3,policy=bind \
 -device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
 -uuid 63840878-0deb-4095-97e6-fc444d9bc9fa \
 -nodefaults \
index 9f0e6963d86ff003d61605aa810f66210e471935..4c11bb762aca80b0860971622683b99fdf35279f 100644 (file)
@@ -11,20 +11,20 @@ QEMU_AUDIO_DRV=none \
 -m 4096 \
 -smp 4,sockets=4,cores=1,threads=1 \
 -object memory-backend-file,id=ram-node0,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=0,cpus=0,memdev=ram-node0 \
 -object memory-backend-file,id=ram-node1,prealloc=yes,\
-mem-path=/dev/hugepages2M/libvirt/qemu,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=1,cpus=1,memdev=ram-node1 \
 -object memory-backend-file,id=ram-node2,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=2,cpus=2,memdev=ram-node2 \
 -object memory-backend-file,id=ram-node3,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=3,policy=bind \
 -numa node,nodeid=3,cpus=3,memdev=ram-node3 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -nographic \
index a37f03d0d2ed9e6a8a5f563259d5b42ea560ff7a..72a1d8c9ba337464bda2bf9adad0d0774b408fae 100644 (file)
@@ -11,7 +11,7 @@ QEMU_AUDIO_DRV=none \
 -m 1024 \
 -smp 2,sockets=2,cores=1,threads=1 \
 -mem-prealloc \
--mem-path /dev/hugepages2M/libvirt/qemu \
+-mem-path /dev/hugepages2M/libvirt/qemu/-1-SomeDummyHugepagesGu \
 -numa node,nodeid=0,cpus=0,mem=256 \
 -numa node,nodeid=1,cpus=1,mem=768 \
 -uuid ef1bdff4-27f3-4e85-a807-5fb4d58463cc \
index 57dd3fa43fc8f1fd12a184b56e5593642e708944..75cb1de59a9bbd31021d247242e9a09cccfb8def 100644 (file)
@@ -13,7 +13,7 @@ QEMU_AUDIO_DRV=none \
 -object memory-backend-ram,id=ram-node0,size=268435456 \
 -numa node,nodeid=0,cpus=0,memdev=ram-node0 \
 -object memory-backend-file,id=ram-node1,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=805306368 \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-SomeDummyHugepagesGu,size=805306368 \
 -numa node,nodeid=1,cpus=1,memdev=ram-node1 \
 -uuid ef1bdff4-27f3-4e85-a807-5fb4d58463cc \
 -nographic \
index a826149791ea751521390b55cd88144dd377faaf..fb9fa8a8fec91305efd61104f7564b8f733315d9 100644 (file)
@@ -10,7 +10,7 @@ QEMU_AUDIO_DRV=none \
 -M pc \
 -m 1024 \
 -mem-prealloc \
--mem-path /dev/hugepages2M/libvirt/qemu \
+-mem-path /dev/hugepages2M/libvirt/qemu/-1-SomeDummyHugepagesGu \
 -smp 2,sockets=2,cores=1,threads=1 \
 -uuid ef1bdff4-27f3-4e85-a807-5fb4d58463cc \
 -nographic \
index f9fc218a763b992074bd46305c2137ff1d233edf..27a6d849a94e61c7caf6dddb9d1ecc2c2eabf35d 100644 (file)
@@ -11,20 +11,20 @@ QEMU_AUDIO_DRV=none \
 -m 4096 \
 -smp 4,sockets=4,cores=1,threads=1 \
 -object memory-backend-file,id=ram-node0,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=0,cpus=0,memdev=ram-node0 \
 -object memory-backend-file,id=ram-node1,prealloc=yes,\
-mem-path=/dev/hugepages2M/libvirt/qemu,share=yes,size=1073741824,\
+mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,share=yes,size=1073741824,\
 host-nodes=0-3,policy=bind \
 -numa node,nodeid=1,cpus=1,memdev=ram-node1 \
 -object memory-backend-file,id=ram-node2,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,share=no,size=1073741824,host-nodes=0-3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,share=no,size=1073741824,\
+host-nodes=0-3,policy=bind \
 -numa node,nodeid=2,cpus=2,memdev=ram-node2 \
 -object memory-backend-file,id=ram-node3,prealloc=yes,\
-mem-path=/dev/hugepages1G/libvirt/qemu,size=1073741824,host-nodes=3,\
-policy=bind \
+mem-path=/dev/hugepages1G/libvirt/qemu/-1-QEMUGuest1,size=1073741824,\
+host-nodes=3,policy=bind \
 -numa node,nodeid=3,cpus=3,memdev=ram-node3 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -nographic \
index 361c8e5998645f1603a7b9fed31cd541c7f81dc8..52d04cdf651692cbf1ff09d5f6e2ba664be9be61 100644 (file)
@@ -10,7 +10,7 @@ QEMU_AUDIO_DRV=none \
 -M pc \
 -m 214 \
 -mem-prealloc \
--mem-path /dev/hugepages2M/libvirt/qemu \
+-mem-path /dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1 \
 -smp 1,sockets=1,cores=1,threads=1 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -nographic \
index fdbb4c32997edfe7b34660e16cb67011ecf476a1..6825fa53325f264dc4d8062b80d602e4858a2038 100644 (file)
@@ -12,8 +12,8 @@ QEMU_AUDIO_DRV=none \
 -smp 2,sockets=2,cores=1,threads=1 \
 -numa node,nodeid=0,cpus=0-1,mem=214 \
 -object memory-backend-file,id=memdimm0,prealloc=yes,\
-mem-path=/dev/hugepages2M/libvirt/qemu,size=536870912,host-nodes=1-3,\
-policy=bind \
+mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,size=536870912,\
+host-nodes=1-3,policy=bind \
 -device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0,addr=4294967296 \
 -object memory-backend-ram,id=memdimm2,size=536870912 \
 -device pc-dimm,node=0,memdev=memdimm2,id=dimm2,slot=2 \
index 1587aba5345edfe1a60b646ce0b68ba392870e74..321e64e23ab30aa8c00ffa8fa7e99ac6dfd7462a 100644 (file)
@@ -14,8 +14,8 @@ QEMU_AUDIO_DRV=none \
 -object memory-backend-ram,id=memdimm0,size=536870912 \
 -device pc-dimm,node=0,memdev=memdimm0,id=dimm0,slot=0 \
 -object memory-backend-file,id=memdimm1,prealloc=yes,\
-mem-path=/dev/hugepages2M/libvirt/qemu,size=536870912,host-nodes=1-3,\
-policy=bind \
+mem-path=/dev/hugepages2M/libvirt/qemu/-1-QEMUGuest1,size=536870912,\
+host-nodes=1-3,policy=bind \
 -device pc-dimm,node=0,memdev=memdimm1,id=dimm1,slot=1 \
 -uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
 -nographic \