From 72cb073c095014f6babab50c20029ba9f2986f1b Mon Sep 17 00:00:00 2001
From: Erik Joelsson <erikj@openjdk.org>
Date: Fri, 6 Dec 2019 12:01:53 -0800
Subject: [PATCH] 8230067: Add optional automatic retry when running jtreg
 tests

Reviewed-by: prr
---
 doc/testing.html          |  2 ++
 doc/testing.md            |  4 ++++
 make/RunTests.gmk         | 50 +++++++++++++++++++++++++++------------
 make/conf/jib-profiles.js |  2 +-
 4 files changed, 42 insertions(+), 16 deletions(-)

diff --git a/doc/testing.html b/doc/testing.html
index 0587be1b237..5b8dde5cecf 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -168,6 +168,8 @@ TEST FAILURE</code></pre>
 <p>Additional VM options to JTReg (<code>-vmoption</code>).</p>
 <h4 id="aot_modules-1">AOT_MODULES</h4>
 <p>Generate AOT modules before testing for the specified module, or set of modules. If multiple modules are specified, they should be separated by space (or, to help avoid quoting issues, the special value <code>%20</code>).</p>
+<h4 id="retry_count">RETRY_COUNT</h4>
+<p>Retry failed tests up to a set number of times. Defaults to 0.</p>
 <h3 id="gtest-keywords">Gtest keywords</h3>
 <h4 id="repeat">REPEAT</h4>
 <p>The number of times to repeat the tests (<code>--gtest_repeat</code>).</p>
diff --git a/doc/testing.md b/doc/testing.md
index 308dcf57daf..ed1c786a00e 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -332,6 +332,10 @@ Generate AOT modules before testing for the specified module, or set of
 modules. If multiple modules are specified, they should be separated by space
 (or, to help avoid quoting issues, the special value `%20`).
 
+#### RETRY_COUNT
+
+Retry failed tests up to a set number of times. Defaults to 0.
+
 ### Gtest keywords
 
 #### REPEAT
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index 916dde0f3cb..92bbe782502 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -300,7 +300,8 @@ $(eval $(call SetTestOpt,FAILURE_HANDLER_TIMEOUT,JTREG))
 
 $(eval $(call ParseKeywordVariable, JTREG, \
     SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR FAILURE_HANDLER_TIMEOUT \
-        TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM RUN_PROBLEM_LISTS, \
+        TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM RUN_PROBLEM_LISTS \
+        RETRY_COUNT, \
     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS KEYWORDS \
         EXTRA_PROBLEM_LISTS AOT_MODULES, \
 ))
@@ -851,6 +852,7 @@ define SetupRunJtregTestBody
   JTREG_VERBOSE ?= fail,error,summary
   JTREG_RETAIN ?= fail,error
   JTREG_RUN_PROBLEM_LISTS ?= false
+  JTREG_RETRY_COUNT ?= 0
 
   ifneq ($$($1_JTREG_MAX_MEM), 0)
     $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
@@ -942,25 +944,43 @@ define SetupRunJtregTestBody
   clean-workdir-$1:
 	$$(RM) -r $$($1_TEST_SUPPORT_DIR)
 
+  $1_COMMAND_LINE := \
+      $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
+          -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
+          $$($1_JTREG_BASIC_OPTIONS) \
+          -testjdk:$$(JDK_UNDER_TEST) \
+          -dir:$$(JTREG_TOPDIR) \
+          -reportDir:$$($1_TEST_RESULTS_DIR) \
+          -workDir:$$($1_TEST_SUPPORT_DIR) \
+          -status:$$$${JTREG_STATUS} \
+          $$(JTREG_OPTIONS) \
+          $$(JTREG_FAILURE_HANDLER_OPTIONS) \
+          $$(JTREG_COV_OPTIONS) \
+          $$($1_TEST_NAME) \
+      && $$(ECHO) $$$$? > $$($1_EXITCODE) \
+      || $$(ECHO) $$$$? > $$($1_EXITCODE)
+
+
+  ifneq ($$(JTREG_RETRY_COUNT), 0)
+    $1_COMMAND_LINE := \
+        for i in {0..$$(JTREG_RETRY_COUNT)}; do \
+          if [ "$$$$i" != 0 ]; then \
+            $$(PRINTF) "\nRetrying Jtreg run. Attempt: $$$$i\n"; \
+          fi; \
+          $$($1_COMMAND_LINE); \
+          if [ "`$$(CAT) $$($1_EXITCODE)`" = "0" ]; then \
+            break; \
+          fi; \
+          export JTREG_STATUS="-status:error,fail"; \
+        done
+  endif
+
   run-test-$1: pre-run-test clean-workdir-$1 $$($1_AOT_TARGETS)
 	$$(call LogWarn)
 	$$(call LogWarn, Running test '$$($1_TEST)')
 	$$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 	$$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, ( \
-	    $$(COV_ENVIRONMENT) \
-	    $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
-	        -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
-	        $$($1_JTREG_BASIC_OPTIONS) \
-	        -testjdk:$$(JDK_UNDER_TEST) \
-	        -dir:$$(JTREG_TOPDIR) \
-	        -reportDir:$$($1_TEST_RESULTS_DIR) \
-	        -workDir:$$($1_TEST_SUPPORT_DIR) \
-	        $$(JTREG_OPTIONS) \
-	        $$(JTREG_FAILURE_HANDLER_OPTIONS) \
-	        $$(JTREG_COV_OPTIONS) \
-	        $$($1_TEST_NAME) \
-	    && $$(ECHO) $$$$? > $$($1_EXITCODE) \
-	    || $$(ECHO) $$$$? > $$($1_EXITCODE) \
+            $$(COV_ENVIRONMENT) $$($1_COMMAND_LINE) \
 	))
 
   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index b8b1a39d738..08dc9aac8dd 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -1043,7 +1043,7 @@ var getJibProfilesDependencies = function (input, common) {
         jtreg: {
             server: "javare",
             revision: "4.2",
-            build_number: "b14",
+            build_number: "b16",
             checksum_file: "MD5_VALUES",
             file: "jtreg_bin-4.2.zip",
             environment_name: "JT_HOME",