MAR-125: Drop common VBD error message dups + use rate limits.

 - Log only the first treq failure, driving vreq->error. Later treqs
   don't contribute to vreq status, so need not be logged either.

 - Ratelimit vreq failures (tlog_drv_error)

 - Store previous errors in vreq->prev_error upon retry. If the
   condition doesn't change during retry, drop the message.

 - Final timeout should be ERR, not LOG_INFO message.

Aims to significantly reduce retry log spam on borked SRs, hopefully
without losing critical information.

Signed-off-by: Daniel Stodden <daniel.stodden@citrix.com>

diff --git a/drivers/tapdisk-vbd.c b/drivers/tapdisk-vbd.c
index 7a32aec8fcecb7c2350c7b5432010b7135fffa6b..708be9dd2685d391631dfcaf4bbc973349ae1db5 100644 (file)
--- a/drivers/tapdisk-vbd.c
+++ b/drivers/tapdisk-vbd.c
@@ -65,7 +65,6 @@
 #define ASSERT(p) ((void)0)
 #endif 
 
-
 #define TD_VBD_EIO_RETRIES          10
 #define TD_VBD_EIO_SLEEP            1
 #define TD_VBD_WATCHDOG_TIMEOUT     10
@@ -826,7 +825,8 @@ __tapdisk_vbd_request_timeout(td_vbd_request_t *vreq,
 
        timeout = tapdisk_vbd_request_ttl(vreq, now) < 0;
        if (timeout)
-               DBG(TLOG_INFO, "req %s timed out, retried %d times\n",
+               ERR(vreq->error,
+                   "req %s timed out, retried %d times\n",
                    vreq->name, vreq->num_retries);
 
        return timeout;
@@ -979,14 +979,17 @@ __tapdisk_vbd_complete_td_request(td_vbd_t *vbd, td_vbd_request_t *vreq,
        }
 
        if (err) {
-               vreq->error = (vreq->error ? : err);
                if (err != -EBUSY) {
+                       if (!vreq->error &&
+                           err != vreq->prev_error)
+                               tlog_drv_error(image->driver, err,
+                                              "req %s: %s 0x%04x secs @ 0x%08"PRIx64,
+                                              vreq->name,
+                                              (treq.op == TD_OP_WRITE ? "write" : "read"),
+                                              treq.secs, treq.sec);
                        vbd->errors++;
-                       ERR(err, "req %s: %s 0x%04x secs @ 0x%08"PRIx64,
-                           vreq->name,
-                           (treq.op == TD_OP_WRITE ? "write" : "read"),
-                           treq.secs, treq.sec);
                }
+               vreq->error = (vreq->error ? : err);
        }
 
        tapdisk_vbd_complete_vbd_request(vbd, vreq);
@@ -1240,7 +1243,10 @@ tapdisk_vbd_reissue_failed_requests(td_vbd_t *vbd)
 
                vbd->retries++;
                vreq->num_retries++;
-               vreq->error  = 0;
+
+               vreq->prev_error = vreq->error;
+               vreq->error      = 0;
+
                DBG(TLOG_DBG, "retry #%d of req %s, "
                    "sec 0x%08"PRIx64", iovcnt: %d\n", vreq->num_retries,
                    vreq->name, vreq->sec, vreq->iovcnt);

diff --git a/drivers/tapdisk.h b/drivers/tapdisk.h
index db1b4f60ec537a752b25a462f2b4ac7af48f26bc..a4c855627139631d4ffc2aefb28937cbb2a38a4b 100644 (file)
--- a/drivers/tapdisk.h
+++ b/drivers/tapdisk.h
@@ -147,6 +147,8 @@ struct td_vbd_request {
        const char                 *name;
 
        int                         error;
+       int                         prev_error;
+
        int                         submitting;
        int                         secs_pending;
        int                         num_retries;