8253920: Share method trampolines in CDS dynamic archive

Reviewed-by: redestad, minqi, iklam

src/hotspot/share/classfile/systemDictionaryShared.cpp

@@ -1179,19 +1179,24 @@ InstanceKlass* SystemDictionaryShared::acquire_class_for_current_thread(
   return shared_klass;
 }
 
-static ResourceHashtable<
+class LoadedUnregisteredClassesTable : public ResourceHashtable<
   Symbol*, bool,
   primitive_hash<Symbol*>,
   primitive_equals<Symbol*>,
   6661,                             // prime number
-  ResourceObj::C_HEAP> _loaded_unregistered_classes;
+  ResourceObj::C_HEAP> {};
+
+static LoadedUnregisteredClassesTable* _loaded_unregistered_classes = NULL;
 
 bool SystemDictionaryShared::add_unregistered_class(InstanceKlass* k, TRAPS) {
   // We don't allow duplicated unregistered classes of the same name.
   assert(DumpSharedSpaces, "only when dumping");
   Symbol* name = k->name();
+  if (_loaded_unregistered_classes == NULL) {
+    _loaded_unregistered_classes = new (ResourceObj::C_HEAP, mtClass)LoadedUnregisteredClassesTable();
+  }
   bool created = false;
-  _loaded_unregistered_classes.put_if_absent(name, true, &created);
+  _loaded_unregistered_classes->put_if_absent(name, true, &created);
   if (created) {
     MutexLocker mu_r(THREAD, Compile_lock); // add_to_hierarchy asserts this.
     SystemDictionary::add_to_hierarchy(k, CHECK_false);

src/hotspot/share/memory/archiveBuilder.cpp

@@ -38,6 +38,7 @@
 #include "oops/instanceKlass.hpp"
 #include "oops/objArrayKlass.hpp"
 #include "oops/oopHandle.inline.hpp"
+#include "runtime/sharedRuntime.hpp"
 #include "utilities/align.hpp"
 #include "utilities/bitMap.inline.hpp"
 #include "utilities/hashtable.inline.hpp"
@@ -45,6 +46,27 @@
 ArchiveBuilder* ArchiveBuilder::_singleton = NULL;
 intx ArchiveBuilder::_buffer_to_target_delta = 0;
 
+class AdapterHandlerEntry;
+
+class MethodTrampolineInfo {
+  address _c2i_entry_trampoline;
+  AdapterHandlerEntry** _adapter_trampoline;
+public:
+  address c2i_entry_trampoline() { return _c2i_entry_trampoline; }
+  AdapterHandlerEntry** adapter_trampoline() { return _adapter_trampoline; }
+  void set_c2i_entry_trampoline(address addr) { _c2i_entry_trampoline = addr; }
+  void set_adapter_trampoline(AdapterHandlerEntry** entry) { _adapter_trampoline = entry; }
+};
+
+class AdapterToTrampoline : public ResourceHashtable<
+  AdapterHandlerEntry*, MethodTrampolineInfo,
+  primitive_hash<AdapterHandlerEntry*>,
+  primitive_equals<AdapterHandlerEntry*>,
+  941, // prime number
+  ResourceObj::C_HEAP> {};
+
+static AdapterToTrampoline* _adapter_to_trampoline = NULL;
+
 ArchiveBuilder::OtherROAllocMark::~OtherROAllocMark() {
   char* newtop = ArchiveBuilder::singleton()->_ro_region->top();
   ArchiveBuilder::alloc_stats()->record_other_type(int(newtop - _oldtop), true);
@@ -259,6 +281,8 @@ void ArchiveBuilder::gather_klasses_and_symbols() {
     // DynamicArchiveBuilder::sort_methods()).
     sort_symbols_and_fix_hash();
     sort_klasses();
+    allocate_method_trampoline_info();
+    allocate_method_trampolines();
   }
 }
 
@@ -798,3 +822,89 @@ void ArchiveBuilder::clean_up_src_obj_table() {
   SrcObjTableCleaner cleaner;
   _src_obj_table.iterate(&cleaner);
 }
+
+void ArchiveBuilder::allocate_method_trampolines_for(InstanceKlass* ik) {
+  if (ik->methods() != NULL) {
+    for (int j = 0; j < ik->methods()->length(); j++) {
+      // Walk the methods in a deterministic order so that the trampolines are
+      // created in a deterministic order.
+      Method* m = ik->methods()->at(j);
+      AdapterHandlerEntry* ent = m->adapter(); // different methods can share the same AdapterHandlerEntry
+      MethodTrampolineInfo* info = _adapter_to_trampoline->get(ent);
+      if (info->c2i_entry_trampoline() == NULL) {
+        info->set_c2i_entry_trampoline(
+          (address)MetaspaceShared::misc_code_space_alloc(SharedRuntime::trampoline_size()));
+        info->set_adapter_trampoline(
+          (AdapterHandlerEntry**)MetaspaceShared::misc_code_space_alloc(sizeof(AdapterHandlerEntry*)));
+      }
+    }
+  }
+}
+
+void ArchiveBuilder::allocate_method_trampolines() {
+  for (int i = 0; i < _klasses->length(); i++) {
+    Klass* k = _klasses->at(i);
+    if (k->is_instance_klass()) {
+      InstanceKlass* ik = InstanceKlass::cast(k);
+      allocate_method_trampolines_for(ik);
+    }
+  }
+}
+
+// Allocate MethodTrampolineInfo for all Methods that will be archived. Also
+// return the total number of bytes needed by the method trampolines in the MC
+// region.
+size_t ArchiveBuilder::allocate_method_trampoline_info() {
+  size_t total = 0;
+  size_t each_method_bytes =
+    align_up(SharedRuntime::trampoline_size(), BytesPerWord) +
+    align_up(sizeof(AdapterHandlerEntry*), BytesPerWord);
+
+  if (_adapter_to_trampoline == NULL) {
+    _adapter_to_trampoline = new (ResourceObj::C_HEAP, mtClass)AdapterToTrampoline();
+  }
+  int count = 0;
+  for (int i = 0; i < _klasses->length(); i++) {
+    Klass* k = _klasses->at(i);
+    if (k->is_instance_klass()) {
+      InstanceKlass* ik = InstanceKlass::cast(k);
+      if (ik->methods() != NULL) {
+        for (int j = 0; j < ik->methods()->length(); j++) {
+          Method* m = ik->methods()->at(j);
+          AdapterHandlerEntry* ent = m->adapter(); // different methods can share the same AdapterHandlerEntry
+          bool is_created = false;
+          MethodTrampolineInfo* info = _adapter_to_trampoline->put_if_absent(ent, &is_created);
+          if (is_created) {
+            count++;
+          }
+        }
+      }
+    }
+  }
+  if (count == 0) {
+    // We have nothing to archive, but let's avoid having an empty region.
+    total = SharedRuntime::trampoline_size();
+  } else {
+    total = count * each_method_bytes;
+  }
+  return align_up(total, SharedSpaceObjectAlignment);
+}
+
+void ArchiveBuilder::update_method_trampolines() {
+  for (int i = 0; i < klasses()->length(); i++) {
+    Klass* k = klasses()->at(i);
+    if (k->is_instance_klass()) {
+      InstanceKlass* ik = InstanceKlass::cast(k);
+      Array<Method*>* methods = ik->methods();
+      for (int j = 0; j < methods->length(); j++) {
+        Method* m = methods->at(j);
+        AdapterHandlerEntry* ent = m->adapter();
+        MethodTrampolineInfo* info = _adapter_to_trampoline->get(ent);
+        // m is the "copy" of the original Method, but its adapter() field is still valid because
+        // we haven't called make_klasses_shareable() yet.
+        m->set_from_compiled_entry(info->c2i_entry_trampoline());
+        m->set_adapter_trampoline(info->adapter_trampoline());
+      }
+    }
+  }
+}

src/hotspot/share/memory/archiveBuilder.hpp

@@ -279,6 +279,13 @@ public:
 
   void print_stats(int ro_all, int rw_all, int mc_all);
   static intx _buffer_to_target_delta;
+
+  // Method trampolines related functions
+  void allocate_method_trampolines();
+  void allocate_method_trampolines_for(InstanceKlass* ik);
+  size_t allocate_method_trampoline_info();
+  void update_method_trampolines();
+
 };
 
 #endif // SHARE_MEMORY_ARCHIVEBUILDER_HPP

src/hotspot/share/memory/dynamicArchive.cpp

@@ -93,12 +93,10 @@ public:
   size_t _estimated_trampoline_bytes;    // method entry trampolines
 
   size_t estimate_archive_size();
-  size_t estimate_trampoline_size();
   size_t estimate_class_file_size();
   address reserve_space_and_init_buffer_to_target_delta();
   void init_header(address addr);
   void release_header();
-  void make_trampolines();
   void sort_methods();
   void sort_methods(InstanceKlass* ik) const;
   void remark_pointers_for_instance_klass(InstanceKlass* k, bool should_mark) const;
@@ -116,12 +114,6 @@ public:
     _num_dump_regions_used = 1;
   }
 
-  void reserve_buffers_for_trampolines() {
-    size_t n = _estimated_trampoline_bytes;
-    assert(n >= SharedRuntime::trampoline_size(), "dont want to be empty");
-    MetaspaceShared::misc_code_space_alloc(n);
-  }
-
 public:
   DynamicArchiveBuilder() : ArchiveBuilder(MetaspaceShared::misc_code_dump_space(),
                                            MetaspaceShared::read_write_dump_space(),
@@ -184,7 +176,7 @@ public:
     CHeapBitMap ptrmap;
     ArchivePtrMarker::initialize(&ptrmap, (address*)reserved_bottom, (address*)current_dump_space()->top());
 
-    reserve_buffers_for_trampolines();
+    allocate_method_trampolines();
     verify_estimate_size(_estimated_trampoline_bytes, "Trampolines");
 
     gather_source_objs();
@@ -221,7 +213,7 @@ public:
 
     verify_estimate_size(_estimated_hashtable_bytes, "Hashtables");
 
-    make_trampolines();
+    update_method_trampolines();
     sort_methods();
 
     log_info(cds)("Make classes shareable");
@@ -254,7 +246,7 @@ size_t DynamicArchiveBuilder::estimate_archive_size() {
   size_t dictionary_est = SystemDictionaryShared::estimate_size_for_archive();
   _estimated_hashtable_bytes = symbol_table_est + dictionary_est;
 
-  _estimated_trampoline_bytes = estimate_trampoline_size();
+  _estimated_trampoline_bytes = allocate_method_trampoline_info();
 
   size_t total = 0;
 
@@ -337,54 +329,6 @@ void DynamicArchiveBuilder::release_header() {
   _header = NULL;
 }
 
-size_t DynamicArchiveBuilder::estimate_trampoline_size() {
-  size_t total = 0;
-  size_t each_method_bytes =
-    align_up(SharedRuntime::trampoline_size(), BytesPerWord) +
-    align_up(sizeof(AdapterHandlerEntry*), BytesPerWord);
-
-  for (int i = 0; i < klasses()->length(); i++) {
-    Klass* k = klasses()->at(i);
-    if (k->is_instance_klass()) {
-      Array<Method*>* methods = InstanceKlass::cast(k)->methods();
-      total += each_method_bytes * methods->length();
-    }
-  }
-  if (total == 0) {
-    // We have nothing to archive, but let's avoid having an empty region.
-    total = SharedRuntime::trampoline_size();
-  }
-  return align_up(total, SharedSpaceObjectAlignment);
-}
-
-void DynamicArchiveBuilder::make_trampolines() {
-  DumpRegion* mc_space = MetaspaceShared::misc_code_dump_space();
-  char* p = mc_space->base();
-  for (int i = 0; i < klasses()->length(); i++) {
-    Klass* k = klasses()->at(i);
-    if (!k->is_instance_klass()) {
-      continue;
-    }
-    InstanceKlass* ik = InstanceKlass::cast(k);
-    Array<Method*>* methods = ik->methods();
-    for (int j = 0; j < methods->length(); j++) {
-      Method* m = methods->at(j);
-      address c2i_entry_trampoline = (address)p;
-      p += SharedRuntime::trampoline_size();
-      assert(p >= mc_space->base() && p <= mc_space->top(), "must be");
-      m->set_from_compiled_entry(to_target(c2i_entry_trampoline));
-
-      AdapterHandlerEntry** adapter_trampoline =(AdapterHandlerEntry**)p;
-      p += sizeof(AdapterHandlerEntry*);
-      assert(p >= mc_space->base() && p <= mc_space->top(), "must be");
-      *adapter_trampoline = NULL;
-      m->set_adapter_trampoline(to_target(adapter_trampoline));
-    }
-  }
-
-  guarantee(p <= mc_space->top(), "Estimate of trampoline size is insufficient");
-}
-
 void DynamicArchiveBuilder::sort_methods() {
   InstanceKlass::disable_method_binary_search();
   for (int i = 0; i < klasses()->length(); i++) {

src/hotspot/share/memory/metaspaceShared.cpp

@@ -770,6 +770,9 @@ void VM_PopulateDumpSharedSpace::doit() {
 
   builder.relocate_well_known_klasses();
 
+  log_info(cds)("Update method trampolines");
+  builder.update_method_trampolines();
+
   log_info(cds)("Make classes shareable");
   builder.make_klasses_shareable();
 

src/hotspot/share/oops/method.cpp

@@ -1107,17 +1107,9 @@ void Method::unlink_method() {
   _i2i_entry = Interpreter::entry_for_cds_method(methodHandle(Thread::current(), this));
   _from_interpreted_entry = _i2i_entry;
 
-  if (DynamicDumpSharedSpaces) {
-    assert(_from_compiled_entry != NULL, "sanity");
-  } else {
-    // TODO: Simplify the adapter trampoline allocation for static archiving.
-    //       Remove the use of CDSAdapterHandlerEntry.
-    CDSAdapterHandlerEntry* cds_adapter = (CDSAdapterHandlerEntry*)adapter();
-    constMethod()->set_adapter_trampoline(cds_adapter->get_adapter_trampoline());
-    _from_compiled_entry = cds_adapter->get_c2i_entry_trampoline();
-    assert(*((int*)_from_compiled_entry) == 0,
-           "must be NULL during dump time, to be initialized at run time");
-  }
+  assert(_from_compiled_entry != NULL, "sanity");
+  assert(*((int*)_from_compiled_entry) == 0,
+         "must be NULL during dump time, to be initialized at run time");
 
   if (is_native()) {
     *native_function_addr() = NULL;

src/hotspot/share/runtime/sharedRuntime.cpp

@@ -2454,15 +2454,12 @@ class AdapterHandlerTable : public BasicHashtable<mtCode> {
 
  public:
   AdapterHandlerTable()
-    : BasicHashtable<mtCode>(293, (DumpSharedSpaces ? sizeof(CDSAdapterHandlerEntry) : sizeof(AdapterHandlerEntry))) { }
+    : BasicHashtable<mtCode>(293, (sizeof(AdapterHandlerEntry))) { }
 
   // Create a new entry suitable for insertion in the table
   AdapterHandlerEntry* new_entry(AdapterFingerPrint* fingerprint, address i2c_entry, address c2i_entry, address c2i_unverified_entry, address c2i_no_clinit_check_entry) {
     AdapterHandlerEntry* entry = (AdapterHandlerEntry*)BasicHashtable<mtCode>::new_entry(fingerprint->compute_hash());
     entry->init(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
-    if (DumpSharedSpaces) {
-      ((CDSAdapterHandlerEntry*)entry)->init();
-    }
     return entry;
   }
 
@@ -3130,17 +3127,6 @@ void AdapterHandlerEntry::print_adapter_on(outputStream* st) const {
   st->cr();
 }
 
-#if INCLUDE_CDS
-
-void CDSAdapterHandlerEntry::init() {
-  assert(DumpSharedSpaces, "used during dump time only");
-  _c2i_entry_trampoline = (address)MetaspaceShared::misc_code_space_alloc(SharedRuntime::trampoline_size());
-  _adapter_trampoline = (AdapterHandlerEntry**)MetaspaceShared::misc_code_space_alloc(sizeof(AdapterHandlerEntry*));
-};
-
-#endif // INCLUDE_CDS
-
-
 #ifndef PRODUCT
 
 void AdapterHandlerLibrary::print_statistics() {

src/hotspot/share/runtime/sharedRuntime.hpp

@@ -679,20 +679,6 @@ class AdapterHandlerEntry : public BasicHashtableEntry<mtCode> {
   void print_adapter_on(outputStream* st) const;
 };
 
-// This class is used only with DumpSharedSpaces==true. It holds extra information
-// that's used only during CDS dump time.
-// For details, see comments around Method::link_method()
-class CDSAdapterHandlerEntry: public AdapterHandlerEntry {
-  address               _c2i_entry_trampoline;   // allocated from shared spaces "MC" region
-  AdapterHandlerEntry** _adapter_trampoline;     // allocated from shared spaces "MD" region
-
-public:
-  address get_c2i_entry_trampoline()             const { return _c2i_entry_trampoline; }
-  AdapterHandlerEntry** get_adapter_trampoline() const { return _adapter_trampoline; }
-  void init() NOT_CDS_RETURN;
-};
-
-
 class AdapterHandlerLibrary: public AllStatic {
  private:
   static BufferBlob* _buffer; // the temporary code buffer in CodeCache