8203664: JFR start failure after AppCDS archive created with JFR StartFlightRecording

Disable JFR and output a warning message during CDS dump time if JFR is enabled.

Reviewed-by: jiangli, lfoltan

src/hotspot/share/jfr/recorder/jfrRecorder.cpp

@@ -181,6 +181,11 @@ static void log_jdk_jfr_module_resolution_error(TRAPS) {
 }
 
 bool JfrRecorder::on_vm_start() {
+  if (DumpSharedSpaces && (JfrOptionSet::startup_recordings() != NULL)) {
+    warning("JFR will be disabled during CDS dumping");
+    teardown_startup_support();
+    return true;
+  }
   const bool in_graph = JfrJavaSupport::is_jdk_jfr_module_available();
   Thread* const thread = Thread::current();
   if (!JfrOptionSet::initialize(thread)) {

src/hotspot/share/jfr/recorder/service/jfrOptionSet.cpp

@@ -707,7 +707,7 @@ void JfrOptionSet::release_startup_recordings() {
     }
   }
   delete startup_recording_array;
-  DEBUG_ONLY(startup_recording_array = NULL;)
+  startup_recording_array = NULL;
 }
 
 bool JfrOptionSet::parse_flight_recorder_option(const JavaVMOption** option, char* tail) {

test/hotspot/jtreg/runtime/appcds/CDSandJFR.java

@@ -74,5 +74,10 @@ public class CDSandJFR {
         TestCommon.checkExec(TestCommon.exec(appJar,
                                              "-XX:FlightRecorderOptions=retransform=false",
                                              "GetFlightRecorder"));
+
+        // Test dumping with flight recorder enabled.
+        output = TestCommon.testDump(appJar, TestCommon.list(classes),
+                                     "-XX:StartFlightRecording=dumponexit=true");
+        TestCommon.checkDump(output, "warning: JFR will be disabled during CDS dumping");
     }
 }