qemu: Adaptive timeout for connecting to monitor

There were couple of reports on the list (e.g. [1]) that guests
with huge amounts of RAM are unable to start because libvirt
kills qemu in the initialization phase. The problem is that if
guest is configured to use hugepages kernel has to zero them all
out before handing over to qemu process. For instance, 402GiB
worth of 1GiB pages took around 105 seconds (~3.8GiB/s). Since we
do not want to make the timeout for connecting to monitor
configurable, we have to teach libvirt to count with this
fact. This commit implements "1s per each 1GiB of RAM" approach
as suggested here [2].

1: https://www.redhat.com/archives/libvir-list/2017-March/msg00373.html
2: https://www.redhat.com/archives/libvir-list/2017-March/msg00405.html

Signed-off-by: Michal Privoznik <mprivozn@redhat.com>

diff --git a/src/qemu/qemu_capabilities.c b/src/qemu/qemu_capabilities.c
index da03ff4c61f885a21df8f576a80b6f7837dd79e3..5200b39a3fbcd0e27b6c4b69bf2cb1c6465fd536 100644 (file)
--- a/src/qemu/qemu_capabilities.c
+++ b/src/qemu/qemu_capabilities.c
@@ -4767,7 +4767,7 @@ virQEMUCapsInitQMPCommandRun(virQEMUCapsInitQMPCommandPtr cmd,
     cmd->vm->pid = cmd->pid;
 
     if (!(cmd->mon = qemuMonitorOpen(cmd->vm, &cmd->config, true,
-                                     &callbacks, NULL)))
+                                     0, &callbacks, NULL)))
         goto ignore;
 
     virObjectLock(cmd->mon);

diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index d71f84c8005aadbd9ccc055a709e36c45a1cfe3b..066d3444bb60779456526dc3aa5b8115e891dfac 100644 (file)
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -327,11 +327,13 @@ qemuMonitorDispose(void *obj)
 
 
 static int
-qemuMonitorOpenUnix(const char *monitor, pid_t cpid)
+qemuMonitorOpenUnix(const char *monitor,
+                    pid_t cpid,
+                    unsigned long long timeout)
 {
     struct sockaddr_un addr;
     int monfd;
-    virTimeBackOffVar timeout;
+    virTimeBackOffVar timebackoff;
     int ret = -1;
 
     if ((monfd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
@@ -348,9 +350,9 @@ qemuMonitorOpenUnix(const char *monitor, pid_t cpid)
         goto error;
     }
 
-    if (virTimeBackOffStart(&timeout, 1, 30*1000 /* ms */) < 0)
+    if (virTimeBackOffStart(&timebackoff, 1, timeout * 1000) < 0)
         goto error;
-    while (virTimeBackOffWait(&timeout)) {
+    while (virTimeBackOffWait(&timebackoff)) {
         ret = connect(monfd, (struct sockaddr *) &addr, sizeof(addr));
 
         if (ret == 0)
@@ -871,10 +873,30 @@ qemuMonitorOpenInternal(virDomainObjPtr vm,
 }
 
 
+#define QEMU_DEFAULT_MONITOR_WAIT 30
+
+/**
+ * qemuMonitorOpen:
+ * @vm: domain object
+ * @config: monitor configuration
+ * @json: enable JSON on the monitor
+ * @timeout: number of seconds to add to default timeout
+ * @cb: monitor event handles
+ * @opaque: opaque data for @cb
+ *
+ * Opens the monitor for running qemu. It may happen that it
+ * takes some time for qemu to create the monitor socket (e.g.
+ * because kernel is zeroing configured hugepages), therefore we
+ * wait up to default + timeout seconds for the monitor to show
+ * up after which a failure is claimed.
+ *
+ * Returns monitor object, NULL on error.
+ */
 qemuMonitorPtr
 qemuMonitorOpen(virDomainObjPtr vm,
                 virDomainChrSourceDefPtr config,
                 bool json,
+                unsigned long long timeout,
                 qemuMonitorCallbacksPtr cb,
                 void *opaque)
 {
@@ -882,10 +904,14 @@ qemuMonitorOpen(virDomainObjPtr vm,
     bool hasSendFD = false;
     qemuMonitorPtr ret;
 
+    timeout += QEMU_DEFAULT_MONITOR_WAIT;
+
     switch (config->type) {
     case VIR_DOMAIN_CHR_TYPE_UNIX:
         hasSendFD = true;
-        if ((fd = qemuMonitorOpenUnix(config->data.nix.path, vm ? vm->pid : 0)) < 0)
+        if ((fd = qemuMonitorOpenUnix(config->data.nix.path,
+                                      vm ? vm->pid : 0,
+                                      timeout)) < 0)
             return NULL;
         break;
 

diff --git a/src/qemu/qemu_monitor.h b/src/qemu/qemu_monitor.h
index 847e9458a554d660e3b2912765349df64e9f0fa7..3c37a6ffe366fa67b91208308d24bff1114396e3 100644 (file)
--- a/src/qemu/qemu_monitor.h
+++ b/src/qemu/qemu_monitor.h
@@ -246,6 +246,7 @@ char *qemuMonitorUnescapeArg(const char *in);
 qemuMonitorPtr qemuMonitorOpen(virDomainObjPtr vm,
                                virDomainChrSourceDefPtr config,
                                bool json,
+                               unsigned long long timeout,
                                qemuMonitorCallbacksPtr cb,
                                void *opaque)
     ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(4);

diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index b9c1847bba31a0ec940c3a138019610dc7d6e0a2..7397f59c7ab75b03a4d20c8ff605fd27d4e871ff 100644 (file)
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -1658,6 +1658,7 @@ qemuConnectMonitor(virQEMUDriverPtr driver, virDomainObjPtr vm, int asyncJob,
     qemuDomainObjPrivatePtr priv = vm->privateData;
     int ret = -1;
     qemuMonitorPtr mon = NULL;
+    unsigned long long timeout = 0;
 
     if (qemuSecuritySetDaemonSocketLabel(driver->securityManager, vm->def) < 0) {
         VIR_ERROR(_("Failed to set security context for monitor for %s"),
@@ -1665,6 +1666,12 @@ qemuConnectMonitor(virQEMUDriverPtr driver, virDomainObjPtr vm, int asyncJob,
         return -1;
     }
 
+    /* When using hugepages, kernel zeroes them out before
+     * handing them over to qemu. This can be very time
+     * consuming. Therefore, add a second to timeout for each
+     * 1GiB of guest RAM. */
+    timeout = vm->def->mem.total_memory / (1024 * 1024);
+
     /* Hold an extra reference because we can't allow 'vm' to be
      * deleted until the monitor gets its own reference. */
     virObjectRef(vm);
@@ -1675,6 +1682,7 @@ qemuConnectMonitor(virQEMUDriverPtr driver, virDomainObjPtr vm, int asyncJob,
     mon = qemuMonitorOpen(vm,
                           priv->monConfig,
                           priv->monJSON,
+                          timeout,
                           &monitorCallbacks,
                           driver);
 

diff --git a/tests/qemumonitortestutils.c b/tests/qemumonitortestutils.c
index cfd0a38cbb0a18bfc9aa3bf56419179da1eb0946..89857a6620ba02f2e707873d5025e752bd7ed307 100644 (file)
--- a/tests/qemumonitortestutils.c
+++ b/tests/qemumonitortestutils.c
@@ -1175,6 +1175,7 @@ qemuMonitorTestNew(bool json,
     if (!(test->mon = qemuMonitorOpen(test->vm,
                                       &src,
                                       json,
+                                      0,
                                       &qemuMonitorTestCallbacks,
                                       driver)))
         goto error;