db_retry: Make the sleeps random and increasing

When there's a thundering herd, this can run out of retries.

Signed-off-by: Ian Jackson <Ian.Jackson@eu.citrix.com>

diff --git a/Osstest.pm b/Osstest.pm
index 734c0ef6bf91d98685c514228ca8d034c5df29b8..809194f0dc2e3be00a4b074f5ba78d06357a6f35 100644 (file)
--- a/Osstest.pm
+++ b/Osstest.pm
@@ -311,7 +311,8 @@ sub db_retry ($$$;$$) {
     my ($pre,$body) =
         (ref $code eq 'ARRAY') ? @$code : (sub { }, $code);
 
-    my $retries= 100;
+    my $max_retries= 100;
+    my $retry_count= 0;
     my $r;
     local $db_retry_stop;
     for (;;) {
@@ -339,10 +340,12 @@ sub db_retry ($$$;$$) {
        };
        last if !length $@;
        die $@ unless $mjobdb->need_retry($dbh, $committing);
-        die "$dbh $body $@ ?" unless $retries-- > 0;
+        die "$dbh $body $@ GIVING UP ?" if ++$retry_count >= $max_retries;
        eval { $dbh->rollback(); };
-       print STDERR "DB conflict (messages above may refer); retrying...\n";
-        sleep(1);
+       my $delay = rand $retry_count;
+       print STDERR "DB conflict (messages above may refer);".
+           " retrying after $delay...\n";
+        sleep($delay);
     }
     return $r;
 }