8232636: JFR TestDumpOnCrash.java crashed and failed to create emergency dump file

Reviewed-by: mgronlun, mseledtsov

test/jdk/jdk/jfr/jvm/TestDumpOnCrash.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,8 +52,7 @@ import jdk.test.lib.process.ProcessTools;
  */
 public class TestDumpOnCrash {
 
-    private static final CharSequence LOG_FILE_EXTENSION = ".log";
-    private static final CharSequence JFR_FILE_EXTENSION = ".jfr";
+    private static final int ATTEMPTS = 3;
 
     static class CrasherIllegalAccess {
         public static void main(String[] args) {
@@ -77,26 +76,46 @@ public class TestDumpOnCrash {
     }
 
     public static void main(String[] args) throws Exception {
-        verify(runProcess(CrasherIllegalAccess.class.getName(), "", true));
-        verify(runProcess(CrasherIllegalAccess.class.getName(), "", false));
-        verify(runProcess(CrasherHalt.class.getName(), "", true));
-        verify(runProcess(CrasherHalt.class.getName(), "", false));
+        test(CrasherIllegalAccess.class, "", true);
+        test(CrasherIllegalAccess.class, "", false);
+        test(CrasherHalt.class, "", true);
+        test(CrasherHalt.class, "", false);
 
-        // Verification is excluded for the test case below until 8219680 is fixed
-        long pid = runProcess(CrasherSig.class.getName(), "FPE", true);
+        // Test is excluded until 8219680 is fixed
         // @ignore 8219680
-        // verify(pid);
+        // test(CrasherSig.class, "FPE", true);
     }
 
-    private static long runProcess(String crasher, String signal, boolean disk) throws Exception {
-        System.out.println("Test case for crasher " + crasher);
+    private static void test(Class<?> crasher, String signal, boolean disk) throws Exception {
+        // The JVM may be in a state it can't recover from, so try three times
+        // before concluding functionality is not working.
+        for (int attempt = 0; attempt < ATTEMPTS; attempt++) {
+            try {
+                verify(runProcess(crasher, signal, disk));
+                return;
+            } catch (Exception e) {
+                System.out.println("Attempt " + attempt + ". Verification failed:");
+                System.out.println(e.getMessage());
+                System.out.println("Retrying...");
+                System.out.println();
+            } catch (OutOfMemoryError | StackOverflowError e) {
+                // Could happen if file is corrupt and parser loops or
+                // tries to allocate more memory than what is available
+                return;
+            }
+        }
+        throw new Exception(ATTEMPTS + " attempts with failure!");
+    }
+
+    private static long runProcess(Class<?> crasher, String signal, boolean disk) throws Exception {
+        System.out.println("Test case for crasher " + crasher.getName());
         final String flightRecordingOptions = "dumponexit=true,disk=" + Boolean.toString(disk);
         Process p = ProcessTools.createJavaProcessBuilder(true,
                 "-Xmx64m",
                 "-XX:-CreateCoredumpOnCrash",
                 "--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED",
                 "-XX:StartFlightRecording=" + flightRecordingOptions,
-                crasher,
+                crasher.getName(),
                 signal)
             .start();
 
@@ -119,5 +138,7 @@ public class TestDumpOnCrash {
         List<RecordedEvent> events = RecordingFile.readAllEvents(file);
         Asserts.assertFalse(events.isEmpty(), "No event found");
         System.out.printf("Found event %s%n", events.get(0).getEventType().getName());
+
+        Files.delete(file);
     }
 }