From: Daniel P. Berrange <berrange@redhat.com>
Date: Tue, 3 Nov 2009 18:34:20 +0000 (-0500)
Subject: Allow timeouts waiting for QEMU job lock
X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=9b6efcfef1e240df0b39ed69c159bb0ae53e1d78;p=libvirt.git

Allow timeouts waiting for QEMU job lock

Some monitor commands may take a very long time to complete. It is
not desirable to block other incoming API calls forever. With this
change, if an existing API call is holding the job lock, additional
API calls will not wait forever. They will time out after a short
period of time, allowing application to retry later.

* include/libvirt/virterror.h, src/util/virterror.c: Add new
  VIR_ERR_OPERATION_TIMEOUT error code
* src/qemu/qemu_driver.c: Change to a timed condition variable
  wait for acquiring the monitor job lock
---

diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h
index 4c28501ac3..dfa23f2444 100644
--- a/include/libvirt/virterror.h
+++ b/include/libvirt/virterror.h
@@ -171,6 +171,7 @@ typedef enum {
     VIR_ERR_INVALID_SECRET, /* invalid secret */
     VIR_ERR_NO_SECRET, /* secret not found */
     VIR_ERR_CONFIG_UNSUPPORTED, /* unsupported configuration construct */
+    VIR_ERR_OPERATION_TIMEOUT, /* timeout occurred during operation */
 } virErrorNumber;
 
 /**
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 0b1fd49df5..245abb7311 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -158,18 +158,36 @@ static void qemuDomainObjPrivateFree(void *data)
  * Upon successful return, the object will have its ref count increased,
  * successful calls must be followed by EndJob eventually
  */
+
+/* Give up waiting for mutex after 30 seconds */
+#define QEMU_JOB_WAIT_TIME (1000ull * 30)
+
 static int qemuDomainObjBeginJob(virDomainObjPtr obj) ATTRIBUTE_RETURN_CHECK;
 static int qemuDomainObjBeginJob(virDomainObjPtr obj)
 {
     qemuDomainObjPrivatePtr priv = obj->privateData;
+    struct timeval now;
+    unsigned long long then;
+
+    if (gettimeofday(&now, NULL) < 0) {
+        virReportSystemError(NULL, errno, "%s",
+                             _("cannot get time of day"));
+        return -1;
+    }
+    then = (now.tv_sec * 1000ull) + (now.tv_usec / 1000);
+    then += QEMU_JOB_WAIT_TIME;
 
     virDomainObjRef(obj);
 
     while (priv->jobActive) {
-        if (virCondWait(&priv->jobCond, &obj->lock) < 0) {
+        if (virCondWaitUntil(&priv->jobCond, &obj->lock, then) < 0) {
             virDomainObjUnref(obj);
-            virReportSystemError(NULL, errno,
-                                 "%s", _("cannot acquire job mutex"));
+            if (errno == ETIMEDOUT)
+                qemudReportError(NULL, NULL, NULL, VIR_ERR_OPERATION_TIMEOUT,
+                                 "%s", _("cannot acquire state change lock"));
+            else
+                virReportSystemError(NULL, errno,
+                                     "%s", _("cannot acquire job mutex"));
             return -1;
         }
     }
@@ -190,15 +208,29 @@ static int qemuDomainObjBeginJobWithDriver(struct qemud_driver *driver,
                                            virDomainObjPtr obj)
 {
     qemuDomainObjPrivatePtr priv = obj->privateData;
+    struct timeval now;
+    unsigned long long then;
+
+    if (gettimeofday(&now, NULL) < 0) {
+        virReportSystemError(NULL, errno, "%s",
+                             _("cannot get time of day"));
+        return -1;
+    }
+    then = (now.tv_sec * 1000ull) + (now.tv_usec / 1000);
+    then += QEMU_JOB_WAIT_TIME;
 
     virDomainObjRef(obj);
     qemuDriverUnlock(driver);
 
     while (priv->jobActive) {
-        if (virCondWait(&priv->jobCond, &obj->lock) < 0) {
+        if (virCondWaitUntil(&priv->jobCond, &obj->lock, then) < 0) {
             virDomainObjUnref(obj);
-            virReportSystemError(NULL, errno,
-                                 "%s", _("cannot acquire job mutex"));
+            if (errno == ETIMEDOUT)
+                qemudReportError(NULL, NULL, NULL, VIR_ERR_OPERATION_TIMEOUT,
+                                 "%s", _("cannot acquire state change lock"));
+            else
+                virReportSystemError(NULL, errno,
+                                     "%s", _("cannot acquire job mutex"));
             return -1;
         }
     }
diff --git a/src/util/virterror.c b/src/util/virterror.c
index c8e8623fd5..00d5b2c3b5 100644
--- a/src/util/virterror.c
+++ b/src/util/virterror.c
@@ -1095,6 +1095,12 @@ virErrorMsg(virErrorNumber error, const char *info)
             else
                 errmsg = _("unsupported configuration: %s");
             break;
+        case VIR_ERR_OPERATION_TIMEOUT:
+            if (info == NULL)
+                errmsg = _("Timed out during operation");
+            else
+                errmsg = _("Timed out during operation: %s");
+            break;
     }
     return (errmsg);
 }