Allow timeouts waiting for QEMU job lock

Some monitor commands may take a very long time to complete. It is
not desirable to block other incoming API calls forever. With this
change, if an existing API call is holding the job lock, additional
API calls will not wait forever. They will time out after a short
period of time, allowing application to retry later.

* include/libvirt/virterror.h, src/util/virterror.c: Add new
  VIR_ERR_OPERATION_TIMEOUT error code
* src/qemu/qemu_driver.c: Change to a timed condition variable
  wait for acquiring the monitor job lock

diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h
index 4c28501ac35f3c2a500cd733bd697ce07f047a9e..dfa23f2444e0546f71cfe11d5b7c1f5abd2deb72 100644 (file)
--- a/include/libvirt/virterror.h
+++ b/include/libvirt/virterror.h
@@ -171,6 +171,7 @@ typedef enum {
     VIR_ERR_INVALID_SECRET, /* invalid secret */
     VIR_ERR_NO_SECRET, /* secret not found */
     VIR_ERR_CONFIG_UNSUPPORTED, /* unsupported configuration construct */
+    VIR_ERR_OPERATION_TIMEOUT, /* timeout occurred during operation */
 } virErrorNumber;
 
 /**

diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 0b1fd49df5a923a8d11ff6243496b6c7fe0b57aa..245abb73113b782f68d6a4b5aa3f05d61db6f711 100644 (file)
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -158,18 +158,36 @@ static void qemuDomainObjPrivateFree(void *data)
  * Upon successful return, the object will have its ref count increased,
  * successful calls must be followed by EndJob eventually
  */
+
+/* Give up waiting for mutex after 30 seconds */
+#define QEMU_JOB_WAIT_TIME (1000ull * 30)
+
 static int qemuDomainObjBeginJob(virDomainObjPtr obj) ATTRIBUTE_RETURN_CHECK;
 static int qemuDomainObjBeginJob(virDomainObjPtr obj)
 {
     qemuDomainObjPrivatePtr priv = obj->privateData;
+    struct timeval now;
+    unsigned long long then;
+
+    if (gettimeofday(&now, NULL) < 0) {
+        virReportSystemError(NULL, errno, "%s",
+                             _("cannot get time of day"));
+        return -1;
+    }
+    then = (now.tv_sec * 1000ull) + (now.tv_usec / 1000);
+    then += QEMU_JOB_WAIT_TIME;
 
     virDomainObjRef(obj);
 
     while (priv->jobActive) {
-        if (virCondWait(&priv->jobCond, &obj->lock) < 0) {
+        if (virCondWaitUntil(&priv->jobCond, &obj->lock, then) < 0) {
             virDomainObjUnref(obj);
-            virReportSystemError(NULL, errno,
-                                 "%s", _("cannot acquire job mutex"));
+            if (errno == ETIMEDOUT)
+                qemudReportError(NULL, NULL, NULL, VIR_ERR_OPERATION_TIMEOUT,
+                                 "%s", _("cannot acquire state change lock"));
+            else
+                virReportSystemError(NULL, errno,
+                                     "%s", _("cannot acquire job mutex"));
             return -1;
         }
     }
@@ -190,15 +208,29 @@ static int qemuDomainObjBeginJobWithDriver(struct qemud_driver *driver,
                                            virDomainObjPtr obj)
 {
     qemuDomainObjPrivatePtr priv = obj->privateData;
+    struct timeval now;
+    unsigned long long then;
+
+    if (gettimeofday(&now, NULL) < 0) {
+        virReportSystemError(NULL, errno, "%s",
+                             _("cannot get time of day"));
+        return -1;
+    }
+    then = (now.tv_sec * 1000ull) + (now.tv_usec / 1000);
+    then += QEMU_JOB_WAIT_TIME;
 
     virDomainObjRef(obj);
     qemuDriverUnlock(driver);
 
     while (priv->jobActive) {
-        if (virCondWait(&priv->jobCond, &obj->lock) < 0) {
+        if (virCondWaitUntil(&priv->jobCond, &obj->lock, then) < 0) {
             virDomainObjUnref(obj);
-            virReportSystemError(NULL, errno,
-                                 "%s", _("cannot acquire job mutex"));
+            if (errno == ETIMEDOUT)
+                qemudReportError(NULL, NULL, NULL, VIR_ERR_OPERATION_TIMEOUT,
+                                 "%s", _("cannot acquire state change lock"));
+            else
+                virReportSystemError(NULL, errno,
+                                     "%s", _("cannot acquire job mutex"));
             return -1;
         }
     }

diff --git a/src/util/virterror.c b/src/util/virterror.c
index c8e8623fd572f1553de4dcf7b445bbdde3a3972a..00d5b2c3b5f2059ddfb758ce7c3366a17d7dbbfa 100644 (file)
--- a/src/util/virterror.c
+++ b/src/util/virterror.c
@@ -1095,6 +1095,12 @@ virErrorMsg(virErrorNumber error, const char *info)
             else
                 errmsg = _("unsupported configuration: %s");
             break;
+        case VIR_ERR_OPERATION_TIMEOUT:
+            if (info == NULL)
+                errmsg = _("Timed out during operation");
+            else
+                errmsg = _("Timed out during operation: %s");
+            break;
     }
     return (errmsg);
 }