8294142: make test should report only on executed tests

Reviewed-by: ihse, djelinski, erikj
This commit is contained in:
Aleksey Shipilev 2022-09-26 08:30:45 +00:00
parent 169a5d48af
commit 91a23d775f
3 changed files with 19 additions and 4 deletions

View File

@ -200,6 +200,8 @@ TEST FAILURE</code></pre>
<p>Retry failed tests up to a set number of times, until they pass. This allows to pass the tests with intermittent failures. Defaults to 0.</p>
<h4 id="repeat_count">REPEAT_COUNT</h4>
<p>Repeat the tests up to a set number of times, stopping at first failure. This helps to reproduce intermittent test failures. Defaults to 0.</p>
<h4 id="report">REPORT</h4>
<p>Use this report style when reporting test results (sent to JTReg as <code>-report</code>). Defaults to <code>executed</code>.</p>
<h3 id="gtest-keywords">Gtest keywords</h3>
<h4 id="repeat">REPEAT</h4>
<p>The number of times to repeat the tests (<code>--gtest_repeat</code>).</p>

View File

@ -456,6 +456,11 @@ Repeat the tests up to a set number of times, stopping at first failure.
This helps to reproduce intermittent test failures.
Defaults to 0.
#### REPORT
Use this report style when reporting test results (sent to JTReg as `-report`).
Defaults to `executed`.
### Gtest keywords
#### REPEAT

View File

@ -196,11 +196,12 @@ $(eval $(call SetTestOpt,JAVA_OPTIONS,JTREG))
$(eval $(call SetTestOpt,JOBS,JTREG))
$(eval $(call SetTestOpt,TIMEOUT_FACTOR,JTREG))
$(eval $(call SetTestOpt,FAILURE_HANDLER_TIMEOUT,JTREG))
$(eval $(call SetTestOpt,REPORT,JTREG))
$(eval $(call ParseKeywordVariable, JTREG, \
SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR FAILURE_HANDLER_TIMEOUT \
TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM RUN_PROBLEM_LISTS \
RETRY_COUNT REPEAT_COUNT MAX_OUTPUT $(CUSTOM_JTREG_SINGLE_KEYWORDS), \
RETRY_COUNT REPEAT_COUNT MAX_OUTPUT REPORT $(CUSTOM_JTREG_SINGLE_KEYWORDS), \
STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS KEYWORDS \
EXTRA_PROBLEM_LISTS LAUNCHER_OPTIONS \
$(CUSTOM_JTREG_STRING_KEYWORDS), \
@ -745,6 +746,7 @@ define SetupRunJtregTestBody
JTREG_RUN_PROBLEM_LISTS ?= false
JTREG_RETRY_COUNT ?= 0
JTREG_REPEAT_COUNT ?= 0
JTREG_REPORT ?= executed
ifneq ($$(JTREG_RETRY_COUNT), 0)
ifneq ($$(JTREG_REPEAT_COUNT), 0)
@ -854,6 +856,7 @@ define SetupRunJtregTestBody
-dir:$$(JTREG_TOPDIR) \
-reportDir:$$($1_TEST_RESULTS_DIR) \
-workDir:$$($1_TEST_SUPPORT_DIR) \
-report:$$$${JTREG_REPORT_ITERATION} \
$$$${JTREG_STATUS} \
$$(JTREG_OPTIONS) \
$$(JTREG_FAILURE_HANDLER_OPTIONS) \
@ -865,6 +868,7 @@ define SetupRunJtregTestBody
ifneq ($$(JTREG_RETRY_COUNT), 0)
$1_COMMAND_LINE := \
export JTREG_REPORT_ITERATION=$$(JTREG_REPORT); \
for i in {0..$$(JTREG_RETRY_COUNT)}; do \
if [ "$$$$i" != 0 ]; then \
$$(PRINTF) "\nRetrying Jtreg run. Attempt: $$$$i\n"; \
@ -874,11 +878,11 @@ define SetupRunJtregTestBody
break; \
fi; \
export JTREG_STATUS="-status:error,fail"; \
export JTREG_REPORT_ITERATION="all-executed"; \
done
endif
ifneq ($$(JTREG_REPEAT_COUNT), 0)
else ifneq ($$(JTREG_REPEAT_COUNT), 0)
$1_COMMAND_LINE := \
export JTREG_REPORT_ITERATION=$$(JTREG_REPORT); \
for i in {1..$$(JTREG_REPEAT_COUNT)}; do \
$$(PRINTF) "\nRepeating Jtreg run: $$$$i out of $$(JTREG_REPEAT_COUNT)\n"; \
$$($1_COMMAND_LINE); \
@ -887,6 +891,10 @@ define SetupRunJtregTestBody
break; \
fi; \
done
else
$1_COMMAND_LINE := \
export JTREG_REPORT_ITERATION=$$(JTREG_REPORT); \
$$($1_COMMAND_LINE);
endif
run-test-$1: pre-run-test clean-workdir-$1