From 6ed3f07f487233718d1f5e08b9609f2be12f1e59 Mon Sep 17 00:00:00 2001
From: Jiangli Zhou <jiangli@openjdk.org>
Date: Tue, 19 Jul 2016 13:52:14 -0400
Subject: [PATCH] 8141341: CDS should be disabled if
 JvmtiExport::should_post_class_file_load_hook() is true

Disable loading shared class if JvmtiExport::should_post_class_file_load_hook is true.

Reviewed-by: iklam, acorn, sspitsyn
---
 .../share/vm/classfile/systemDictionary.cpp   | 24 ++++---
 hotspot/src/share/vm/memory/metaspace.cpp     | 62 +++++++++++--------
 2 files changed, 52 insertions(+), 34 deletions(-)

diff --git a/hotspot/src/share/vm/classfile/systemDictionary.cpp b/hotspot/src/share/vm/classfile/systemDictionary.cpp
index b747fae54f2..f9167b98d38 100644
--- a/hotspot/src/share/vm/classfile/systemDictionary.cpp
+++ b/hotspot/src/share/vm/classfile/systemDictionary.cpp
@@ -1246,12 +1246,16 @@ Klass* SystemDictionary::find_shared_class(Symbol* class_name) {
 
 instanceKlassHandle SystemDictionary::load_shared_class(
                  Symbol* class_name, Handle class_loader, TRAPS) {
-  instanceKlassHandle ik (THREAD, find_shared_class(class_name));
-  // Make sure we only return the boot class for the NULL classloader.
-  if (ik.not_null() &&
-      ik->is_shared_boot_class() && class_loader.is_null()) {
-    Handle protection_domain;
-    return load_shared_class(ik, class_loader, protection_domain, THREAD);
+  // Don't load shared class when JvmtiExport::should_post_class_file_load_hook()
+  // is enabled since posting CFLH is not supported when loading shared class.
+  if (!JvmtiExport::should_post_class_file_load_hook()) {
+    instanceKlassHandle ik (THREAD, find_shared_class(class_name));
+    // Make sure we only return the boot class for the NULL classloader.
+    if (ik.not_null() &&
+        ik->is_shared_boot_class() && class_loader.is_null()) {
+      Handle protection_domain;
+      return load_shared_class(ik, class_loader, protection_domain, THREAD);
+    }
   }
   return instanceKlassHandle();
 }
@@ -1334,8 +1338,14 @@ bool SystemDictionary::is_shared_class_visible(Symbol* class_name,
 instanceKlassHandle SystemDictionary::load_shared_class(instanceKlassHandle ik,
                                                         Handle class_loader,
                                                         Handle protection_domain, TRAPS) {
+  instanceKlassHandle nh = instanceKlassHandle(); // null Handle
+  if (JvmtiExport::should_post_class_file_load_hook()) {
+    // Don't load shared class when JvmtiExport::should_post_class_file_load_hook()
+    // is enabled since posting CFLH is not supported when loading shared class.
+    return nh;
+  }
+
   if (ik.not_null()) {
-    instanceKlassHandle nh = instanceKlassHandle(); // null Handle
     Symbol* class_name = ik->name();
 
     bool visible = is_shared_class_visible(
diff --git a/hotspot/src/share/vm/memory/metaspace.cpp b/hotspot/src/share/vm/memory/metaspace.cpp
index 5dce2678527..ac7d0f838a3 100644
--- a/hotspot/src/share/vm/memory/metaspace.cpp
+++ b/hotspot/src/share/vm/memory/metaspace.cpp
@@ -3164,39 +3164,47 @@ void Metaspace::global_initialize() {
 #endif // _LP64
 #endif // INCLUDE_CDS
   } else {
-    // If using shared space, open the file that contains the shared space
-    // and map in the memory before initializing the rest of metaspace (so
-    // the addresses don't conflict)
-    address cds_address = NULL;
-    if (UseSharedSpaces) {
 #if INCLUDE_CDS
+    if (UseSharedSpaces) {
+      // If using shared space, open the file that contains the shared space
+      // and map in the memory before initializing the rest of metaspace (so
+      // the addresses don't conflict)
+      address cds_address = NULL;
       FileMapInfo* mapinfo = new FileMapInfo();
 
-      // Open the shared archive file, read and validate the header. If
-      // initialization fails, shared spaces [UseSharedSpaces] are
-      // disabled and the file is closed.
-      // Map in spaces now also
-      if (mapinfo->initialize() && MetaspaceShared::map_shared_spaces(mapinfo)) {
-        cds_total = FileMapInfo::shared_spaces_size();
-        cds_address = (address)mapinfo->header()->region_addr(0);
-#ifdef _LP64
-        if (using_class_space()) {
-          char* cds_end = (char*)(cds_address + cds_total);
-          cds_end = (char *)align_ptr_up(cds_end, _reserve_alignment);
-          // If UseCompressedClassPointers is set then allocate the metaspace area
-          // above the heap and above the CDS area (if it exists).
-          allocate_metaspace_compressed_klass_ptrs(cds_end, cds_address);
-          // Map the shared string space after compressed pointers
-          // because it relies on compressed class pointers setting to work
-          mapinfo->map_string_regions();
-        }
-#endif // _LP64
+      if (JvmtiExport::should_post_class_file_load_hook()) {
+        // Currently CDS does not support JVMTI CFLH when loading shared class.
+        // If JvmtiExport::should_post_class_file_load_hook is already enabled,
+        // just disable UseSharedSpaces.
+        FileMapInfo::fail_continue("Tool agent requires sharing to be disabled.");
+        delete mapinfo;
       } else {
-        assert(!mapinfo->is_open() && !UseSharedSpaces,
-               "archive file not closed or shared spaces not disabled.");
+        // Open the shared archive file, read and validate the header. If
+        // initialization fails, shared spaces [UseSharedSpaces] are
+        // disabled and the file is closed.
+        // Map in spaces now also
+        if (mapinfo->initialize() && MetaspaceShared::map_shared_spaces(mapinfo)) {
+          cds_total = FileMapInfo::shared_spaces_size();
+          cds_address = (address)mapinfo->header()->region_addr(0);
+#ifdef _LP64
+          if (using_class_space()) {
+            char* cds_end = (char*)(cds_address + cds_total);
+            cds_end = (char *)align_ptr_up(cds_end, _reserve_alignment);
+            // If UseCompressedClassPointers is set then allocate the metaspace area
+            // above the heap and above the CDS area (if it exists).
+            allocate_metaspace_compressed_klass_ptrs(cds_end, cds_address);
+            // Map the shared string space after compressed pointers
+            // because it relies on compressed class pointers setting to work
+            mapinfo->map_string_regions();
+          }
+#endif // _LP64
+        } else {
+          assert(!mapinfo->is_open() && !UseSharedSpaces,
+                 "archive file not closed or shared spaces not disabled.");
+        }
       }
-#endif // INCLUDE_CDS
     }
+#endif // INCLUDE_CDS
 
 #ifdef _LP64
     if (!UseSharedSpaces && using_class_space()) {