From 91a23d775fbf482244ace5758f7b3084ea564460 Mon Sep 17 00:00:00 2001
From: Aleksey Shipilev <shade@openjdk.org>
Date: Mon, 26 Sep 2022 08:30:45 +0000
Subject: [PATCH] 8294142: make test should report only on executed tests

Reviewed-by: ihse, djelinski, erikj
---
 doc/testing.html  |  2 ++
 doc/testing.md    |  5 +++++
 make/RunTests.gmk | 16 ++++++++++++----
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/doc/testing.html b/doc/testing.html
index 1ef3038d194..9d90d13e357 100644
--- a/doc/testing.html
+++ b/doc/testing.html
@@ -200,6 +200,8 @@ TEST FAILURE</code></pre>
 <p>Retry failed tests up to a set number of times, until they pass. This allows to pass the tests with intermittent failures. Defaults to 0.</p>
 <h4 id="repeat_count">REPEAT_COUNT</h4>
 <p>Repeat the tests up to a set number of times, stopping at first failure. This helps to reproduce intermittent test failures. Defaults to 0.</p>
+<h4 id="report">REPORT</h4>
+<p>Use this report style when reporting test results (sent to JTReg as <code>-report</code>). Defaults to <code>executed</code>.</p>
 <h3 id="gtest-keywords">Gtest keywords</h3>
 <h4 id="repeat">REPEAT</h4>
 <p>The number of times to repeat the tests (<code>--gtest_repeat</code>).</p>
diff --git a/doc/testing.md b/doc/testing.md
index 5cac75ca2c8..634c240aeda 100644
--- a/doc/testing.md
+++ b/doc/testing.md
@@ -456,6 +456,11 @@ Repeat the tests up to a set number of times, stopping at first failure.
 This helps to reproduce intermittent test failures.
 Defaults to 0.
 
+#### REPORT
+
+Use this report style when reporting test results (sent to JTReg as `-report`).
+Defaults to `executed`.
+
 ### Gtest keywords
 
 #### REPEAT
diff --git a/make/RunTests.gmk b/make/RunTests.gmk
index c9ee216a722..edf294eaba9 100644
--- a/make/RunTests.gmk
+++ b/make/RunTests.gmk
@@ -196,11 +196,12 @@ $(eval $(call SetTestOpt,JAVA_OPTIONS,JTREG))
 $(eval $(call SetTestOpt,JOBS,JTREG))
 $(eval $(call SetTestOpt,TIMEOUT_FACTOR,JTREG))
 $(eval $(call SetTestOpt,FAILURE_HANDLER_TIMEOUT,JTREG))
+$(eval $(call SetTestOpt,REPORT,JTREG))
 
 $(eval $(call ParseKeywordVariable, JTREG, \
     SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR FAILURE_HANDLER_TIMEOUT \
         TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM RUN_PROBLEM_LISTS \
-        RETRY_COUNT REPEAT_COUNT MAX_OUTPUT $(CUSTOM_JTREG_SINGLE_KEYWORDS), \
+        RETRY_COUNT REPEAT_COUNT MAX_OUTPUT REPORT $(CUSTOM_JTREG_SINGLE_KEYWORDS), \
     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS KEYWORDS \
         EXTRA_PROBLEM_LISTS LAUNCHER_OPTIONS \
         $(CUSTOM_JTREG_STRING_KEYWORDS), \
@@ -745,6 +746,7 @@ define SetupRunJtregTestBody
   JTREG_RUN_PROBLEM_LISTS ?= false
   JTREG_RETRY_COUNT ?= 0
   JTREG_REPEAT_COUNT ?= 0
+  JTREG_REPORT ?= executed
 
   ifneq ($$(JTREG_RETRY_COUNT), 0)
     ifneq ($$(JTREG_REPEAT_COUNT), 0)
@@ -854,6 +856,7 @@ define SetupRunJtregTestBody
           -dir:$$(JTREG_TOPDIR) \
           -reportDir:$$($1_TEST_RESULTS_DIR) \
           -workDir:$$($1_TEST_SUPPORT_DIR) \
+          -report:$$$${JTREG_REPORT_ITERATION} \
           $$$${JTREG_STATUS} \
           $$(JTREG_OPTIONS) \
           $$(JTREG_FAILURE_HANDLER_OPTIONS) \
@@ -865,6 +868,7 @@ define SetupRunJtregTestBody
 
   ifneq ($$(JTREG_RETRY_COUNT), 0)
     $1_COMMAND_LINE := \
+        export JTREG_REPORT_ITERATION=$$(JTREG_REPORT); \
         for i in {0..$$(JTREG_RETRY_COUNT)}; do \
           if [ "$$$$i" != 0 ]; then \
             $$(PRINTF) "\nRetrying Jtreg run. Attempt: $$$$i\n"; \
@@ -874,11 +878,11 @@ define SetupRunJtregTestBody
             break; \
           fi; \
           export JTREG_STATUS="-status:error,fail"; \
+          export JTREG_REPORT_ITERATION="all-executed"; \
         done
-  endif
-
-  ifneq ($$(JTREG_REPEAT_COUNT), 0)
+  else ifneq ($$(JTREG_REPEAT_COUNT), 0)
     $1_COMMAND_LINE := \
+        export JTREG_REPORT_ITERATION=$$(JTREG_REPORT); \
         for i in {1..$$(JTREG_REPEAT_COUNT)}; do \
           $$(PRINTF) "\nRepeating Jtreg run: $$$$i out of $$(JTREG_REPEAT_COUNT)\n"; \
           $$($1_COMMAND_LINE); \
@@ -887,6 +891,10 @@ define SetupRunJtregTestBody
             break; \
           fi; \
         done
+  else
+    $1_COMMAND_LINE := \
+        export JTREG_REPORT_ITERATION=$$(JTREG_REPORT); \
+        $$($1_COMMAND_LINE);
   endif
 
   run-test-$1: pre-run-test clean-workdir-$1