8325536: JVM crash during CDS archive creation with -XX:+AllowArchivingWithJavaAgent

Reviewed-by: iklam, matsaave

src/hotspot/share/cds/archiveBuilder.cpp

@@ -740,10 +740,6 @@ void ArchiveBuilder::make_klasses_shareable() {
       assert(k->is_instance_klass(), " must be");
       num_instance_klasses ++;
       InstanceKlass* ik = InstanceKlass::cast(k);
-      if (CDSConfig::is_dumping_dynamic_archive()) {
-        // For static dump, class loader type are already set.
-        ik->assign_class_loader_type();
-      }
       if (ik->is_shared_boot_class()) {
         type = "boot";
         num_boot_klasses ++;

src/hotspot/share/classfile/klassFactory.cpp

@@ -97,6 +97,7 @@ InstanceKlass* KlassFactory::check_shared_class_file_load_hook(
 
       if (class_loader.is_null()) {
         new_ik->set_classpath_index(path_index);
+        new_ik->assign_class_loader_type();
       }
 
       return new_ik;

test/hotspot/jtreg/runtime/cds/appcds/jvmti/dumpingWithAgent/AppWithBMH.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+// Application which loads BoundMethodHandle species classes like the following:
+// java/lang/invoke/BoundMethodHandle$Species_LLLL
+import java.lang.management.ManagementFactory;
+
+public class AppWithBMH {
+  public static void main(String[] args) {
+    System.out.println("Hello world!");
+    ManagementFactory.getGarbageCollectorMXBeans();
+  }
+}

test/hotspot/jtreg/runtime/cds/appcds/jvmti/dumpingWithAgent/DumpingWithJavaAgent.java

@@ -28,7 +28,7 @@
  * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds /test/hotspot/jtreg/runtime/cds/appcds/test-classes
  * @requires vm.cds
  * @requires vm.jvmti
- * @build SimpleAgent Hello
+ * @build SimpleAgent Hello AppWithBMH
  * @run main/othervm DumpingWithJavaAgent
  */
 
@@ -40,9 +40,11 @@ import jdk.test.lib.helpers.ClassFileInstaller;
 public class DumpingWithJavaAgent {
     public static String appClasses[] = {
         "Hello",
+        "AppWithBMH",
     };
     public static String agentClasses[] = {
         "SimpleAgent",
+        "SimpleAgent$1"
     };
 
     public static String warningMessages[] = {
@@ -65,8 +67,18 @@ public class DumpingWithJavaAgent {
         String appJar =
             ClassFileInstaller.writeJar("DumpingWithJavaAgent.jar", appClasses);
 
+        // CDS dumping with a java agent performing class transformation on BoundMethodHandle$Species classes
+        OutputAnalyzer output = TestCommon.testDump(appJar, TestCommon.list("AppWithBMH"),
+            "-XX:+UnlockDiagnosticVMOptions", diagnosticOption,
+            "-javaagent:" + agentJar + "=doTransform",
+            "AppWithBMH");
+        TestCommon.checkDump(output);
+        output.shouldContain(warningMessages[0]);
+        output.shouldContain(warningMessages[1]);
+        output.shouldContain("inside SimpleAgent");
+
         // CDS dumping with a java agent with the AllowArchvingWithJavaAgent diagnostic option.
-        OutputAnalyzer output = TestCommon.testDump(appJar, TestCommon.list("Hello"),
+        output = TestCommon.testDump(appJar, TestCommon.list("Hello"),
             "-XX:+UnlockDiagnosticVMOptions", diagnosticOption,
             "-javaagent:" + agentJar);
         TestCommon.checkDump(output);

test/hotspot/jtreg/runtime/cds/appcds/jvmti/dumpingWithAgent/SimpleAgent.java

@@ -21,14 +21,27 @@
  * questions.
  *
  */
+import java.lang.instrument.ClassFileTransformer;
 import java.lang.instrument.Instrumentation;
+import java.security.ProtectionDomain;
+import java.util.Arrays;
 
 public class SimpleAgent {
     public static void premain(String agentArg, Instrumentation instrumentation) throws Exception {
         System.out.println("inside SimpleAgent");
-        // Only load the class if the test requires it.
-        if (agentArg != null && agentArg.equals("OldSuper")) {
+        if (agentArg == null) return;
+        if (agentArg.equals("OldSuper")) {
+            // Only load the class if the test requires it.
             Class<?> cls = Class.forName("OldSuper", true, ClassLoader.getSystemClassLoader());
+        } else if (agentArg.equals("doTransform")) {
+            ClassFileTransformer transformer = new ClassFileTransformer() {
+            @Override
+            public byte[] transform(ClassLoader loader, String className, Class<?> classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) {
+                 System.out.printf("%n Transforming %s", className);
+                 return Arrays.copyOf(classfileBuffer, classfileBuffer.length);
+               }
+             };
+             instrumentation.addTransformer(transformer);
         }
     }
 }