8213250: CDS archive creation aborts due to metaspace object allocation failure

Reviewed-by: jiangli, ccheung

src/hotspot/share/classfile/systemDictionaryShared.cpp

@@ -27,6 +27,7 @@
 #include "classfile/classListParser.hpp"
 #include "classfile/classLoader.hpp"
 #include "classfile/classLoaderData.inline.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
 #include "classfile/classLoaderExt.hpp"
 #include "classfile/dictionary.hpp"
 #include "classfile/javaClasses.hpp"
@@ -782,8 +783,16 @@ bool SystemDictionaryShared::add_verification_constraint(InstanceKlass* k, Symbo
   }
 }
 
+void SystemDictionaryShared::finalize_verification_constraints_for(InstanceKlass* k) {
+  if (!k->is_unsafe_anonymous()) {
+    SharedDictionaryEntry* entry = ((SharedDictionary*)(k->class_loader_data()->dictionary()))->find_entry_for(k);
+    entry->finalize_verification_constraints();
+  }
+}
+
 void SystemDictionaryShared::finalize_verification_constraints() {
-  boot_loader_dictionary()->finalize_verification_constraints();
+  MutexLocker mcld(ClassLoaderDataGraph_lock);
+  ClassLoaderDataGraph::dictionary_classes_do(finalize_verification_constraints_for);
 }
 
 void SystemDictionaryShared::check_verification_constraints(InstanceKlass* klass,
@@ -810,28 +819,6 @@ SharedDictionaryEntry* SharedDictionary::find_entry_for(InstanceKlass* klass) {
   return NULL;
 }
 
-void SharedDictionary::finalize_verification_constraints() {
-  int bytes = 0, count = 0;
-  for (int index = 0; index < table_size(); index++) {
-    for (SharedDictionaryEntry *probe = bucket(index);
-                                probe != NULL;
-                               probe = probe->next()) {
-      int n = probe->finalize_verification_constraints();
-      if (n > 0) {
-        bytes += n;
-        count ++;
-      }
-    }
-  }
-  if (log_is_enabled(Info, cds, verification)) {
-    double avg = 0;
-    if (count > 0) {
-      avg = double(bytes) / double(count);
-    }
-    log_info(cds, verification)("Recorded verification constraints for %d classes = %d bytes (avg = %.2f bytes) ", count, bytes, avg);
-  }
-}
-
 void SharedDictionaryEntry::add_verification_constraint(Symbol* name,
          Symbol* from_name, bool from_field_is_protected, bool from_is_array, bool from_is_object) {
   if (_verifier_constraints == NULL) {

src/hotspot/share/classfile/systemDictionaryShared.hpp

@@ -172,7 +172,6 @@ class SharedDictionary : public Dictionary {
 
 public:
   SharedDictionaryEntry* find_entry_for(InstanceKlass* klass);
-  void finalize_verification_constraints();
 
   bool add_non_builtin_klass(const Symbol* class_name,
                              ClassLoaderData* loader_data,
@@ -283,7 +282,7 @@ private:
                                  Handle class_loader,
                                  Handle protection_domain,
                                  TRAPS);
-
+  static void finalize_verification_constraints_for(InstanceKlass* k);
 public:
   // Called by PLATFORM/APP loader only
   static InstanceKlass* find_or_load_shared_class(Symbol* class_name,

src/hotspot/share/memory/metaspace.cpp

@@ -1260,6 +1260,8 @@ size_t Metaspace::align_word_size_up(size_t word_size) {
 MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
                               MetaspaceObj::Type type, TRAPS) {
   assert(!_frozen, "sanity");
+  assert(!(DumpSharedSpaces && THREAD->is_VM_thread()), "sanity");
+
   if (HAS_PENDING_EXCEPTION) {
     assert(false, "Should not allocate with exception pending");
     return NULL;  // caller does a CHECK_NULL too
@@ -1277,12 +1279,10 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
     tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype);
 
     // Allocation failed.
-    if (is_init_completed() && !(DumpSharedSpaces && THREAD->is_VM_thread())) {
+    if (is_init_completed()) {
       // Only start a GC if the bootstrapping has completed.
-      // Also, we cannot GC if we are at the end of the CDS dumping stage which runs inside
-      // the VM thread.
-
-      // Try to clean out some memory and retry.
+      // Try to clean out some heap memory and retry. This can prevent premature
+      // expansion of the metaspace.
       result = Universe::heap()->satisfy_failed_metadata_allocation(loader_data, word_size, mdtype);
     }
   }
@@ -1408,6 +1408,7 @@ ClassLoaderMetaspace::ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType
 }
 
 ClassLoaderMetaspace::~ClassLoaderMetaspace() {
+  Metaspace::assert_not_frozen();
   DEBUG_ONLY(Atomic::inc(&g_internal_statistics.num_metaspace_deaths));
   delete _vsm;
   if (Metaspace::using_class_space()) {

src/hotspot/share/memory/metaspaceShared.cpp

@@ -1207,10 +1207,6 @@ private:
 
 public:
   static void copy_and_compact() {
-    // We should no longer allocate anything from the metaspace, so that
-    // we can have a stable set of MetaspaceObjs to work with.
-    Metaspace::freeze();
-
     ResourceMark rm;
     SortedSymbolClosure the_ssc; // StackObj
     _ssc = &the_ssc;
@@ -1356,6 +1352,14 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables() {
 }
 
 void VM_PopulateDumpSharedSpace::doit() {
+  // We should no longer allocate anything from the metaspace, so that:
+  //
+  // (1) Metaspace::allocate might trigger GC if we have run out of
+  //     committed metaspace, but we can't GC because we're running
+  //     in the VM thread.
+  // (2) ArchiveCompactor needs to work with a stable set of MetaspaceObjs.
+  Metaspace::freeze();
+
   Thread* THREAD = VMThread::vm_thread();
 
   FileMapInfo::check_nonempty_dir_in_shared_path_table();
@@ -1420,8 +1424,6 @@ void VM_PopulateDumpSharedSpace::doit() {
   // any dictionaries.
   NOT_PRODUCT(assert_no_unsafe_anonymous_classes_in_dictionaries());
 
-  SystemDictionaryShared::finalize_verification_constraints();
-
   ArchiveCompactor::initialize();
   ArchiveCompactor::copy_and_compact();
 
@@ -1717,6 +1719,8 @@ void MetaspaceShared::preload_and_dump(TRAPS) {
 
     SystemDictionary::clear_invoke_method_table();
 
+    SystemDictionaryShared::finalize_verification_constraints();
+
     VM_PopulateDumpSharedSpace op;
     VMThread::execute(&op);
   }