From: Ian Jackson <ian.jackson@eu.citrix.com>
Date: Thu, 16 Jul 2020 14:51:41 +0000 (+0100)
Subject: cr-daily-branch: Immediately retry failing tests
X-Git-Url: http://xenbits.xensource.com/gitweb?a=commitdiff_plain;h=fab07c777da46faf4c2d9d62de605b183695e89d;p=osstest.git

cr-daily-branch: Immediately retry failing tests

We exclude the self-tests because we don't want to miss breakage, and
the Xen smoke tests because they will be run again RSN anyway.

Signed-off-by: Ian Jackson <iwj@xenproject.org>
---

diff --git a/cr-daily-branch b/cr-daily-branch
index 285ea36..bea8734 100755
--- a/cr-daily-branch
+++ b/cr-daily-branch
@@ -472,12 +472,58 @@ esac
 sgr_args+=" $EXTRA_SGR_ARGS"
 
 date >&2
+original_start=`date +%s`
+
 : $flight $branch $OSSTEST_BLESSING $sgr_args
 eval "$DAILY_BRANCH_PREEXEC_HOOK"
 execute_flight $flight $OSSTEST_BLESSING
 date >&2
 
-start_email $flight $branch "$sgr_args" "$subject_prefix"
+default_immediate_retry=$wantpush
+
+case "$branch" in
+*smoke*)	default_immediate_retry=false ;;
+osstest)	default_immediate_retry=false ;;
+*)		;;
+esac
+
+: ${OSSTEST_IMMEDIATE_RETRY:-$default_immediate_retry}
+
+while true; do
+	start_email $flight $branch "$sgr_args" "$subject_prefix"
+	if grep '^tolerable$' $mrof >/dev/null 2>&1; then break; fi
+	if ! $OSSTEST_IMMEDIATE_RETRY; then break; fi
+	OSSTEST_IMMEDIATE_RETRY=false
+	retry_jobs=$(
+		perl <$mrof -wne '
+			next unless m/^regression (\S+) /;
+			my $j = $1;
+			next if $j =~ m/^build/;
+			$r{$j}++;
+			END {
+				print "copy-jobs '$flight' $_ "
+					foreach sort keys %r;
+			}'
+	)
+	if [ "x$retry_jobs" = x ]; then break; fi
+
+	rflight=$(
+		./cs-adjust-flight new:$OSSTEST_BLESSING \
+			branch-set $branch \
+			$retry_jobs
+	)
+
+	./mg-adjust-flight-makexrefs -v $rflight \
+		--branch=$branch --revision-osstest=$narness_rev \
+		'^build-*' --debug --blessings=real
+
+	export OSSTEST_RESOURCE_WAITSTART=$original_start
+	execute_flight $rflight $OSSTEST_BLESSING-retest
+	report_flight $rflight
+	publish_logs $rflight
+
+	sgr_args+=" --refer-to-flight=$rflight"
+done
 
 push=false
 if grep '^tolerable$' $mrof >/dev/null 2>&1; then push=$wantpush; fi