From dc00eb87bc28ed5bf499af6835c3df474c454a41 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Mon, 16 Sep 2024 05:33:40 +0000 Subject: [PATCH] 8338912: CDS: Segmented roots array Reviewed-by: ccheung, iklam --- src/hotspot/share/cds/archiveBuilder.cpp | 19 +-- src/hotspot/share/cds/archiveHeapLoader.cpp | 14 +- src/hotspot/share/cds/archiveHeapWriter.cpp | 136 ++++++++++++-------- src/hotspot/share/cds/archiveHeapWriter.hpp | 22 +--- src/hotspot/share/cds/archiveUtils.cpp | 21 +++ src/hotspot/share/cds/archiveUtils.hpp | 41 ++++++ src/hotspot/share/cds/filemap.cpp | 8 +- src/hotspot/share/cds/filemap.hpp | 10 +- src/hotspot/share/cds/heapShared.cpp | 44 +++++-- src/hotspot/share/cds/heapShared.hpp | 9 +- 10 files changed, 221 insertions(+), 103 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 76b6698a400..3c67216d4c5 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -1104,6 +1104,17 @@ class ArchiveBuilder::CDSMapLogger : AllStatic { LogStreamHandle(Info, cds, map) st; + HeapRootSegments segments = heap_info->heap_root_segments(); + assert(segments.base_offset() == 0, "Sanity"); + + for (size_t seg_idx = 0; seg_idx < segments.count(); seg_idx++) { + address requested_start = ArchiveHeapWriter::buffered_addr_to_requested_addr(start); + st.print_cr(PTR_FORMAT ": Heap roots segment [%d]", + p2i(requested_start), segments.size_in_elems(seg_idx)); + start += segments.size_in_bytes(seg_idx); + } + log_heap_roots(); + while (start < end) { size_t byte_size; oop source_oop = ArchiveHeapWriter::buffered_addr_to_source_obj(start); @@ -1114,12 +1125,6 @@ class ArchiveBuilder::CDSMapLogger : AllStatic { // This is a regular oop that got archived. print_oop_with_requested_addr_cr(&st, source_oop, false); byte_size = source_oop->size() * BytesPerWord; - } else if (start == ArchiveHeapWriter::buffered_heap_roots_addr()) { - // HeapShared::roots() is copied specially, so it doesn't exist in - // ArchiveHeapWriter::BufferOffsetToSourceObjectTable. - // See ArchiveHeapWriter::copy_roots_to_buffer(). - st.print_cr("HeapShared::roots[%d]", HeapShared::pending_roots()->length()); - byte_size = ArchiveHeapWriter::heap_roots_word_size() * BytesPerWord; } else if ((byte_size = ArchiveHeapWriter::get_filler_size_at(start)) > 0) { // We have a filler oop, which also does not exist in BufferOffsetToSourceObjectTable. st.print_cr("filler " SIZE_FORMAT " bytes", byte_size); @@ -1132,8 +1137,6 @@ class ArchiveBuilder::CDSMapLogger : AllStatic { if (source_oop != nullptr) { log_oop_details(heap_info, source_oop, /*buffered_addr=*/start); - } else if (start == ArchiveHeapWriter::buffered_heap_roots_addr()) { - log_heap_roots(); } start = oop_end; } diff --git a/src/hotspot/share/cds/archiveHeapLoader.cpp b/src/hotspot/share/cds/archiveHeapLoader.cpp index feaf245d22c..6325fb6f49d 100644 --- a/src/hotspot/share/cds/archiveHeapLoader.cpp +++ b/src/hotspot/share/cds/archiveHeapLoader.cpp @@ -374,8 +374,18 @@ void ArchiveHeapLoader::finish_initialization() { if (is_in_use()) { patch_native_pointers(); intptr_t bottom = is_loaded() ? _loaded_heap_bottom : _mapped_heap_bottom; - intptr_t roots_oop = bottom + FileMapInfo::current_info()->heap_roots_offset(); - HeapShared::init_roots(cast_to_oop(roots_oop)); + + // The heap roots are stored in one or more segments that are laid out consecutively. + // The byte size of each segment (except for the last one) is max_size. + HeapRootSegments segments = FileMapInfo::current_info()->heap_root_segments(); + int max_size = segments.max_size_in_bytes(); + HeapShared::init_root_segment_sizes(max_size); + intptr_t first_segment_addr = bottom + segments.base_offset(); + for (size_t c = 0; c < segments.count(); c++) { + oop segment_oop = cast_to_oop(first_segment_addr + (c * max_size)); + assert(segment_oop->is_objArray(), "Must be"); + HeapShared::add_root_segment((objArrayOop)segment_oop); + } } } diff --git a/src/hotspot/share/cds/archiveHeapWriter.cpp b/src/hotspot/share/cds/archiveHeapWriter.cpp index bf49805658c..d8ee7155452 100644 --- a/src/hotspot/share/cds/archiveHeapWriter.cpp +++ b/src/hotspot/share/cds/archiveHeapWriter.cpp @@ -52,9 +52,9 @@ GrowableArrayCHeap* ArchiveHeapWriter::_buffer = nullptr; // The following are offsets from buffer_bottom() size_t ArchiveHeapWriter::_buffer_used; -size_t ArchiveHeapWriter::_heap_roots_offset; -size_t ArchiveHeapWriter::_heap_roots_word_size; +// Heap root segments +HeapRootSegments ArchiveHeapWriter::_heap_root_segments; address ArchiveHeapWriter::_requested_bottom; address ArchiveHeapWriter::_requested_top; @@ -164,10 +164,6 @@ address ArchiveHeapWriter::buffered_addr_to_requested_addr(address buffered_addr return _requested_bottom + buffered_address_to_offset(buffered_addr); } -oop ArchiveHeapWriter::heap_roots_requested_address() { - return cast_to_oop(_requested_bottom + _heap_roots_offset); -} - address ArchiveHeapWriter::requested_address() { assert(_buffer != nullptr, "must be initialized"); return _requested_bottom; @@ -186,47 +182,71 @@ void ArchiveHeapWriter::ensure_buffer_space(size_t min_bytes) { _buffer->at_grow(to_array_index(min_bytes)); } +objArrayOop ArchiveHeapWriter::allocate_root_segment(size_t offset, int element_count) { + HeapWord* mem = offset_to_buffered_address(offset); + memset(mem, 0, objArrayOopDesc::object_size(element_count)); + + // The initialization code is copied from MemAllocator::finish and ObjArrayAllocator::initialize. + oopDesc::set_mark(mem, markWord::prototype()); + oopDesc::release_set_klass(mem, Universe::objectArrayKlass()); + arrayOopDesc::set_length(mem, element_count); + return objArrayOop(cast_to_oop(mem)); +} + +void ArchiveHeapWriter::root_segment_at_put(objArrayOop segment, int index, oop root) { + // Do not use arrayOop->obj_at_put(i, o) as arrayOop is outside the real heap! + if (UseCompressedOops) { + *segment->obj_at_addr(index) = CompressedOops::encode(root); + } else { + *segment->obj_at_addr(index) = root; + } +} + void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeap* roots) { - Klass* k = Universe::objectArrayKlass(); // already relocated to point to archived klass - int length = roots->length(); - _heap_roots_word_size = objArrayOopDesc::object_size(length); - size_t byte_size = _heap_roots_word_size * HeapWordSize; - if (byte_size >= MIN_GC_REGION_ALIGNMENT) { - log_error(cds, heap)("roots array is too large. Please reduce the number of classes"); - vm_exit(1); - } + // Depending on the number of classes we are archiving, a single roots array may be + // larger than MIN_GC_REGION_ALIGNMENT. Roots are allocated first in the buffer, which + // allows us to chop the large array into a series of "segments". Current layout + // starts with zero or more segments exactly fitting MIN_GC_REGION_ALIGNMENT, and end + // with a single segment that may be smaller than MIN_GC_REGION_ALIGNMENT. + // This is simple and efficient. We do not need filler objects anywhere between the segments, + // or immediately after the last segment. This allows starting the object dump immediately + // after the roots. - maybe_fill_gc_region_gap(byte_size); + assert((_buffer_used % MIN_GC_REGION_ALIGNMENT) == 0, + "Pre-condition: Roots start at aligned boundary: " SIZE_FORMAT, _buffer_used); - size_t new_used = _buffer_used + byte_size; - ensure_buffer_space(new_used); + int max_elem_count = ((MIN_GC_REGION_ALIGNMENT - arrayOopDesc::header_size_in_bytes()) / heapOopSize); + assert(objArrayOopDesc::object_size(max_elem_count)*HeapWordSize == MIN_GC_REGION_ALIGNMENT, + "Should match exactly"); - HeapWord* mem = offset_to_buffered_address(_buffer_used); - memset(mem, 0, byte_size); - { - // This is copied from MemAllocator::finish - oopDesc::set_mark(mem, markWord::prototype()); - oopDesc::release_set_klass(mem, k); - } - { - // This is copied from ObjArrayAllocator::initialize - arrayOopDesc::set_length(mem, length); - } + HeapRootSegments segments(_buffer_used, + roots->length(), + MIN_GC_REGION_ALIGNMENT, + max_elem_count); - objArrayOop arrayOop = objArrayOop(cast_to_oop(mem)); - for (int i = 0; i < length; i++) { - // Do not use arrayOop->obj_at_put(i, o) as arrayOop is outside of the real heap! - oop o = roots->at(i); - if (UseCompressedOops) { - * arrayOop->obj_at_addr(i) = CompressedOops::encode(o); - } else { - * arrayOop->obj_at_addr(i) = o; + for (size_t seg_idx = 0; seg_idx < segments.count(); seg_idx++) { + int size_elems = segments.size_in_elems(seg_idx); + size_t size_bytes = segments.size_in_bytes(seg_idx); + + size_t oop_offset = _buffer_used; + _buffer_used = oop_offset + size_bytes; + ensure_buffer_space(_buffer_used); + + assert((oop_offset % MIN_GC_REGION_ALIGNMENT) == 0, + "Roots segment " SIZE_FORMAT " start is not aligned: " SIZE_FORMAT, + segments.count(), oop_offset); + + int root_index = 0; + objArrayOop seg_oop = allocate_root_segment(oop_offset, size_elems); + for (int i = 0; i < size_elems; i++) { + root_segment_at_put(seg_oop, i, roots->at(root_index++)); } - } - log_info(cds, heap)("archived obj roots[%d] = " SIZE_FORMAT " bytes, klass = %p, obj = %p", length, byte_size, k, mem); - _heap_roots_offset = _buffer_used; - _buffer_used = new_used; + log_info(cds, heap)("archived obj root segment [%d] = " SIZE_FORMAT " bytes, obj = " PTR_FORMAT, + size_elems, size_bytes, p2i(seg_oop)); + } + + _heap_root_segments = segments; } static int oop_sorting_rank(oop o) { @@ -282,6 +302,10 @@ void ArchiveHeapWriter::sort_source_objs() { } void ArchiveHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeap* roots) { + // There could be multiple root segments, which we want to be aligned by region. + // Putting them ahead of objects makes sure we waste no space. + copy_roots_to_buffer(roots); + sort_source_objs(); for (int i = 0; i < _source_objs_order->length(); i++) { int src_obj_index = _source_objs_order->at(i)._index; @@ -295,8 +319,6 @@ void ArchiveHeapWriter::copy_source_objs_to_buffer(GrowableArrayCHeapmaybe_grow(); } - copy_roots_to_buffer(roots); - log_info(cds)("Size of heap region = " SIZE_FORMAT " bytes, %d objects, %d roots, %d native ptrs", _buffer_used, _source_objs->length() + 1, roots->length(), _num_native_ptrs); } @@ -455,7 +477,7 @@ void ArchiveHeapWriter::set_requested_address(ArchiveHeapInfo* info) { info->set_buffer_region(MemRegion(offset_to_buffered_address(0), offset_to_buffered_address(_buffer_used))); - info->set_heap_roots_offset(_heap_roots_offset); + info->set_heap_root_segments(_heap_root_segments); } // Oop relocation @@ -543,12 +565,6 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s } } -// Relocate an element in the buffered copy of HeapShared::roots() -template void ArchiveHeapWriter::relocate_root_at(oop requested_roots, int index, CHeapBitMap* oopmap) { - size_t offset = (size_t)((objArrayOop)requested_roots)->obj_at_offset(index); - relocate_field_in_buffer((T*)(buffered_heap_roots_addr() + offset), oopmap); -} - class ArchiveHeapWriter::EmbeddedOopRelocator: public BasicOopIterateClosure { oop _src_obj; address _buffered_obj; @@ -600,14 +616,24 @@ void ArchiveHeapWriter::relocate_embedded_oops(GrowableArrayCHeaplength() : 0; - for (int i = 0; i < length; i++) { + for (size_t seg_idx = 0; seg_idx < _heap_root_segments.count(); seg_idx++) { + size_t seg_offset = _heap_root_segments.segment_offset(seg_idx); + + objArrayOop requested_obj = (objArrayOop)requested_obj_from_buffer_offset(seg_offset); + update_header_for_requested_obj(requested_obj, nullptr, Universe::objectArrayKlass()); + address buffered_obj = offset_to_buffered_address
(seg_offset); + int length = _heap_root_segments.size_in_elems(seg_idx); + if (UseCompressedOops) { - relocate_root_at(requested_roots, i, heap_info->oopmap()); + for (int i = 0; i < length; i++) { + narrowOop* addr = (narrowOop*)(buffered_obj + objArrayOopDesc::obj_at_offset(i)); + relocate_field_in_buffer(addr, heap_info->oopmap()); + } } else { - relocate_root_at(requested_roots, i, heap_info->oopmap()); + for (int i = 0; i < length; i++) { + oop* addr = (oop*)(buffered_obj + objArrayOopDesc::obj_at_offset(i)); + relocate_field_in_buffer(addr, heap_info->oopmap()); + } } } diff --git a/src/hotspot/share/cds/archiveHeapWriter.hpp b/src/hotspot/share/cds/archiveHeapWriter.hpp index 352aeb9a08f..961d2b52133 100644 --- a/src/hotspot/share/cds/archiveHeapWriter.hpp +++ b/src/hotspot/share/cds/archiveHeapWriter.hpp @@ -41,8 +41,7 @@ class ArchiveHeapInfo { MemRegion _buffer_region; // Contains the archived objects to be written into the CDS archive. CHeapBitMap _oopmap; CHeapBitMap _ptrmap; - size_t _heap_roots_offset; // Offset of the HeapShared::roots() object, from the bottom - // of the archived heap objects, in bytes. + HeapRootSegments _heap_root_segments; public: ArchiveHeapInfo() : _buffer_region(), _oopmap(128, mtClassShared), _ptrmap(128, mtClassShared) {} @@ -57,8 +56,8 @@ public: CHeapBitMap* oopmap() { return &_oopmap; } CHeapBitMap* ptrmap() { return &_ptrmap; } - void set_heap_roots_offset(size_t n) { _heap_roots_offset = n; } - size_t heap_roots_offset() const { return _heap_roots_offset; } + void set_heap_root_segments(HeapRootSegments segments) { _heap_root_segments = segments; }; + HeapRootSegments heap_root_segments() { return _heap_root_segments; } }; #if INCLUDE_CDS_JAVA_HEAP @@ -130,9 +129,8 @@ private: // The number of bytes that have written into _buffer (may be smaller than _buffer->length()). static size_t _buffer_used; - // The bottom of the copy of Heap::roots() inside this->_buffer. - static size_t _heap_roots_offset; - static size_t _heap_roots_word_size; + // The heap root segments information. + static HeapRootSegments _heap_root_segments; // The address range of the requested location of the archived heap objects. static address _requested_bottom; @@ -193,6 +191,8 @@ private: return buffered_addr - buffer_bottom(); } + static void root_segment_at_put(objArrayOop segment, int index, oop root); + static objArrayOop allocate_root_segment(size_t offset, int element_count); static void copy_roots_to_buffer(GrowableArrayCHeap* roots); static void copy_source_objs_to_buffer(GrowableArrayCHeap* roots); static size_t copy_one_source_obj_to_buffer(oop src_obj); @@ -219,7 +219,6 @@ private: template static T* requested_addr_to_buffered_addr(T* p); template static void relocate_field_in_buffer(T* field_addr_in_buffer, CHeapBitMap* oopmap); template static void mark_oop_pointer(T* buffered_addr, CHeapBitMap* oopmap); - template static void relocate_root_at(oop requested_roots, int index, CHeapBitMap* oopmap); static void update_header_for_requested_obj(oop requested_obj, oop src_obj, Klass* src_klass); @@ -234,13 +233,6 @@ public: static bool is_string_too_large_to_archive(oop string); static void write(GrowableArrayCHeap*, ArchiveHeapInfo* heap_info); static address requested_address(); // requested address of the lowest achived heap object - static oop heap_roots_requested_address(); // requested address of HeapShared::roots() - static address buffered_heap_roots_addr() { - return offset_to_buffered_address
(_heap_roots_offset); - } - static size_t heap_roots_word_size() { - return _heap_roots_word_size; - } static size_t get_filler_size_at(address buffered_addr); static void mark_native_pointer(oop src_obj, int offset); diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp index 76cfa441fa7..4622a27cbec 100644 --- a/src/hotspot/share/cds/archiveUtils.cpp +++ b/src/hotspot/share/cds/archiveUtils.cpp @@ -369,3 +369,24 @@ void ArchiveUtils::log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) { } } } + +size_t HeapRootSegments::size_in_bytes(size_t seg_idx) { + assert(seg_idx < _count, "In range"); + return objArrayOopDesc::object_size(size_in_elems(seg_idx)) * HeapWordSize; +} + +int HeapRootSegments::size_in_elems(size_t seg_idx) { + assert(seg_idx < _count, "In range"); + if (seg_idx != _count - 1) { + return _max_size_in_elems; + } else { + // Last slice, leftover + return _roots_count % _max_size_in_elems; + } +} + +size_t HeapRootSegments::segment_offset(size_t seg_idx) { + assert(seg_idx < _count, "In range"); + return _base_offset + seg_idx * _max_size_in_bytes; +} + diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp index 32cef97886f..2e361ab0c46 100644 --- a/src/hotspot/share/cds/archiveUtils.hpp +++ b/src/hotspot/share/cds/archiveUtils.hpp @@ -250,4 +250,45 @@ public: static void log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) NOT_CDS_RETURN; }; +class HeapRootSegments { +private: + size_t _base_offset; + size_t _count; + int _roots_count; + int _max_size_in_bytes; + int _max_size_in_elems; + +public: + size_t base_offset() { return _base_offset; } + size_t count() { return _count; } + int roots_count() { return _roots_count; } + int max_size_in_bytes() { return _max_size_in_bytes; } + int max_size_in_elems() { return _max_size_in_elems; } + + size_t size_in_bytes(size_t seg_idx); + int size_in_elems(size_t seg_idx); + size_t segment_offset(size_t seg_idx); + + // Trivial copy assignments are allowed to copy the entire object representation. + // We also inline this class into archive header. Therefore, it is important to make + // sure any gaps in object representation are initialized to zeroes. This is why + // constructors memset before doing field assignments. + HeapRootSegments() { + memset(this, 0, sizeof(*this)); + } + HeapRootSegments(size_t base_offset, int roots_count, int max_size_in_bytes, int max_size_in_elems) { + assert(is_power_of_2(max_size_in_bytes), "must be"); + memset(this, 0, sizeof(*this)); + _base_offset = base_offset; + _count = (roots_count + max_size_in_elems - 1) / max_size_in_elems; + _roots_count = roots_count; + _max_size_in_bytes = max_size_in_bytes; + _max_size_in_elems = max_size_in_elems; + } + + // This class is trivially copyable and assignable. + HeapRootSegments(const HeapRootSegments&) = default; + HeapRootSegments& operator=(const HeapRootSegments&) = default; +}; + #endif // SHARE_CDS_ARCHIVEUTILS_HPP diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp index 35c43157b1a..b86118c6868 100644 --- a/src/hotspot/share/cds/filemap.cpp +++ b/src/hotspot/share/cds/filemap.cpp @@ -289,7 +289,11 @@ void FileMapHeader::print(outputStream* st) { st->print_cr("- has_non_jar_in_classpath: %d", _has_non_jar_in_classpath); st->print_cr("- requested_base_address: " INTPTR_FORMAT, p2i(_requested_base_address)); st->print_cr("- mapped_base_address: " INTPTR_FORMAT, p2i(_mapped_base_address)); - st->print_cr("- heap_roots_offset: " SIZE_FORMAT, _heap_roots_offset); + st->print_cr("- heap_root_segments.roots_count: %d" , _heap_root_segments.roots_count()); + st->print_cr("- heap_root_segments.base_offset: " SIZE_FORMAT_X, _heap_root_segments.base_offset()); + st->print_cr("- heap_root_segments.count: " SIZE_FORMAT, _heap_root_segments.count()); + st->print_cr("- heap_root_segments.max_size_elems: %d", _heap_root_segments.max_size_in_elems()); + st->print_cr("- heap_root_segments.max_size_bytes: %d", _heap_root_segments.max_size_in_bytes()); st->print_cr("- _heap_oopmap_start_pos: " SIZE_FORMAT, _heap_oopmap_start_pos); st->print_cr("- _heap_ptrmap_start_pos: " SIZE_FORMAT, _heap_ptrmap_start_pos); st->print_cr("- _rw_ptrmap_start_pos: " SIZE_FORMAT, _rw_ptrmap_start_pos); @@ -1647,7 +1651,7 @@ size_t FileMapInfo::write_heap_region(ArchiveHeapInfo* heap_info) { char* buffer_start = heap_info->buffer_start(); size_t buffer_size = heap_info->buffer_byte_size(); write_region(MetaspaceShared::hp, buffer_start, buffer_size, false, false); - header()->set_heap_roots_offset(heap_info->heap_roots_offset()); + header()->set_heap_root_segments(heap_info->heap_root_segments()); return buffer_size; } diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp index 7b10c16920b..aa728b9d494 100644 --- a/src/hotspot/share/cds/filemap.hpp +++ b/src/hotspot/share/cds/filemap.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_CDS_FILEMAP_HPP #define SHARE_CDS_FILEMAP_HPP +#include "cds/archiveUtils.hpp" #include "cds/metaspaceShared.hpp" #include "include/cds.h" #include "logging/logLevel.hpp" @@ -225,8 +226,7 @@ private: bool _use_optimized_module_handling;// No module-relation VM options were specified, so we can skip // some expensive operations. bool _has_full_module_graph; // Does this CDS archive contain the full archived module graph? - size_t _heap_roots_offset; // Offset of the HeapShared::roots() object, from the bottom - // of the archived heap objects, in bytes. + HeapRootSegments _heap_root_segments; // Heap root segments info size_t _heap_oopmap_start_pos; // The first bit in the oopmap corresponds to this position in the heap. size_t _heap_ptrmap_start_pos; // The first bit in the ptrmap corresponds to this position in the heap. size_t _rw_ptrmap_start_pos; // The first bit in the ptrmap corresponds to this position in the rw region @@ -270,7 +270,7 @@ public: bool has_non_jar_in_classpath() const { return _has_non_jar_in_classpath; } bool compressed_oops() const { return _compressed_oops; } bool compressed_class_pointers() const { return _compressed_class_ptrs; } - size_t heap_roots_offset() const { return _heap_roots_offset; } + HeapRootSegments heap_root_segments() const { return _heap_root_segments; } size_t heap_oopmap_start_pos() const { return _heap_oopmap_start_pos; } size_t heap_ptrmap_start_pos() const { return _heap_ptrmap_start_pos; } size_t rw_ptrmap_start_pos() const { return _rw_ptrmap_start_pos; } @@ -285,7 +285,7 @@ public: void set_cloned_vtables(char* p) { set_as_offset(p, &_cloned_vtables_offset); } void set_serialized_data(char* p) { set_as_offset(p, &_serialized_data_offset); } void set_mapped_base_address(char* p) { _mapped_base_address = p; } - void set_heap_roots_offset(size_t n) { _heap_roots_offset = n; } + void set_heap_root_segments(HeapRootSegments segments) { _heap_root_segments = segments; } void set_heap_oopmap_start_pos(size_t n) { _heap_oopmap_start_pos = n; } void set_heap_ptrmap_start_pos(size_t n) { _heap_ptrmap_start_pos = n; } void set_rw_ptrmap_start_pos(size_t n) { _rw_ptrmap_start_pos = n; } @@ -385,7 +385,7 @@ public: address narrow_oop_base() const { return header()->narrow_oop_base(); } int narrow_oop_shift() const { return header()->narrow_oop_shift(); } uintx max_heap_size() const { return header()->max_heap_size(); } - size_t heap_roots_offset() const { return header()->heap_roots_offset(); } + HeapRootSegments heap_root_segments() const { return header()->heap_root_segments(); } size_t core_region_alignment() const { return header()->core_region_alignment(); } size_t heap_oopmap_start_pos() const { return header()->heap_oopmap_start_pos(); } size_t heap_ptrmap_start_pos() const { return header()->heap_ptrmap_start_pos(); } diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp index abfc0f9d64b..68be14a46fe 100644 --- a/src/hotspot/share/cds/heapShared.cpp +++ b/src/hotspot/share/cds/heapShared.cpp @@ -133,7 +133,9 @@ static ArchivableStaticFieldInfo fmg_archive_subgraph_entry_fields[] = { KlassSubGraphInfo* HeapShared::_default_subgraph_info; GrowableArrayCHeap* HeapShared::_pending_roots = nullptr; -OopHandle HeapShared::_roots; +GrowableArrayCHeap* HeapShared::_root_segments; +int HeapShared::_root_segment_max_size_shift; +int HeapShared::_root_segment_max_size_mask; OopHandle HeapShared::_scratch_basic_type_mirrors[T_VOID+1]; MetaspaceObjToOopHandleTable* HeapShared::_scratch_java_mirror_table = nullptr; MetaspaceObjToOopHandleTable* HeapShared::_scratch_references_table = nullptr; @@ -225,7 +227,7 @@ int HeapShared::append_root(oop obj) { return _pending_roots->append(obj); } -objArrayOop HeapShared::roots() { +objArrayOop HeapShared::root_segment(int segment_idx) { if (CDSConfig::is_dumping_heap()) { assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread"); if (!HeapShared::can_write()) { @@ -235,17 +237,21 @@ objArrayOop HeapShared::roots() { assert(CDSConfig::is_using_archive(), "must be"); } - objArrayOop roots = (objArrayOop)_roots.resolve(); - assert(roots != nullptr, "should have been initialized"); - return roots; + objArrayOop segment = (objArrayOop)_root_segments->at(segment_idx).resolve(); + assert(segment != nullptr, "should have been initialized"); + return segment; } // Returns an objArray that contains all the roots of the archived objects oop HeapShared::get_root(int index, bool clear) { + assert(_root_segment_max_size_shift > 0, "sanity"); + assert(_root_segment_max_size_mask > 0, "sanity"); assert(index >= 0, "sanity"); assert(!CDSConfig::is_dumping_heap() && CDSConfig::is_using_archive(), "runtime only"); - assert(!_roots.is_empty(), "must have loaded shared heap"); - oop result = roots()->obj_at(index); + assert(!_root_segments->is_empty(), "must have loaded shared heap"); + int seg_idx = index >> _root_segment_max_size_shift; + int int_idx = index & _root_segment_max_size_mask; + oop result = root_segment(seg_idx)->obj_at(int_idx); if (clear) { clear_root(index); } @@ -256,11 +262,15 @@ void HeapShared::clear_root(int index) { assert(index >= 0, "sanity"); assert(CDSConfig::is_using_archive(), "must be"); if (ArchiveHeapLoader::is_in_use()) { + assert(_root_segment_max_size_shift > 0, "sanity"); + assert(_root_segment_max_size_mask > 0, "sanity"); + int seg_idx = index >> _root_segment_max_size_shift; + int int_idx = index & _root_segment_max_size_mask; if (log_is_enabled(Debug, cds, heap)) { - oop old = roots()->obj_at(index); + oop old = root_segment(seg_idx)->obj_at(int_idx); log_debug(cds, heap)("Clearing root %d: was " PTR_FORMAT, index, p2i(old)); } - roots()->obj_at_put(index, nullptr); + root_segment(seg_idx)->obj_at_put(int_idx, nullptr); } } @@ -764,11 +774,19 @@ void HeapShared::write_subgraph_info_table() { } } -void HeapShared::init_roots(oop roots_oop) { - if (roots_oop != nullptr) { - assert(ArchiveHeapLoader::is_in_use(), "must be"); - _roots = OopHandle(Universe::vm_global(), roots_oop); +void HeapShared::add_root_segment(objArrayOop segment_oop) { + assert(segment_oop != nullptr, "must be"); + assert(ArchiveHeapLoader::is_in_use(), "must be"); + if (_root_segments == nullptr) { + _root_segments = new GrowableArrayCHeap(10); } + _root_segments->push(OopHandle(Universe::vm_global(), segment_oop)); +} + +void HeapShared::init_root_segment_sizes(int max_size) { + assert(is_power_of_2(max_size), "must be"); + _root_segment_max_size_shift = log2i_exact(max_size); + _root_segment_max_size_mask = max_size - 1; } void HeapShared::serialize_tables(SerializeClosure* soc) { diff --git a/src/hotspot/share/cds/heapShared.hpp b/src/hotspot/share/cds/heapShared.hpp index 0ba20f1e313..9bb85db0fe9 100644 --- a/src/hotspot/share/cds/heapShared.hpp +++ b/src/hotspot/share/cds/heapShared.hpp @@ -290,7 +290,9 @@ private: static KlassSubGraphInfo* _default_subgraph_info; static GrowableArrayCHeap* _pending_roots; - static OopHandle _roots; + static GrowableArrayCHeap* _root_segments; + static int _root_segment_max_size_shift; + static int _root_segment_max_size_mask; static OopHandle _scratch_basic_type_mirrors[T_VOID+1]; static MetaspaceObjToOopHandleTable* _scratch_java_mirror_table; static MetaspaceObjToOopHandleTable* _scratch_references_table; @@ -399,7 +401,7 @@ private: static GrowableArrayCHeap* pending_roots() { return _pending_roots; } // Dump-time and runtime - static objArrayOop roots(); + static objArrayOop root_segment(int segment_idx); static oop get_root(int index, bool clear=false); // Run-time only @@ -422,7 +424,8 @@ private: static void init_for_dumping(TRAPS) NOT_CDS_JAVA_HEAP_RETURN; static void write_subgraph_info_table() NOT_CDS_JAVA_HEAP_RETURN; - static void init_roots(oop roots_oop) NOT_CDS_JAVA_HEAP_RETURN; + static void add_root_segment(objArrayOop segment_oop) NOT_CDS_JAVA_HEAP_RETURN; + static void init_root_segment_sizes(int max_size) NOT_CDS_JAVA_HEAP_RETURN; static void serialize_tables(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN; #ifndef PRODUCT