diff --git a/src/hotspot/share/gc/shared/memAllocator.hpp b/src/hotspot/share/gc/shared/memAllocator.hpp index 2bc3d601720..2d50648ac75 100644 --- a/src/hotspot/share/gc/shared/memAllocator.hpp +++ b/src/hotspot/share/gc/shared/memAllocator.hpp @@ -56,12 +56,16 @@ protected: _word_size(word_size) { } - // This function clears the memory of the object + // Initialization provided by subclasses. + virtual oop initialize(HeapWord* mem) const = 0; + + // This function clears the memory of the object. void mem_clear(HeapWord* mem) const; + // This finish constructing an oop by installing the mark word and the Klass* pointer // last. At the point when the Klass pointer is initialized, this is a constructed object // that must be parseable as an oop by concurrent collectors. - virtual oop finish(HeapWord* mem) const; + oop finish(HeapWord* mem) const; // Raw memory allocation. This will try to do a TLAB allocation, and otherwise fall // back to calling CollectedHeap::mem_allocate(). @@ -72,9 +76,9 @@ protected: } public: + // Allocate and fully construct the object, and perform various instrumentation. Could safepoint. oop allocate() const; oop try_allocate_in_existing_tlab(); - virtual oop initialize(HeapWord* mem) const = 0; }; class ObjAllocator: public MemAllocator { @@ -85,9 +89,10 @@ public: }; class ObjArrayAllocator: public MemAllocator { +protected: const int _length; const bool _do_zero; -protected: + virtual MemRegion obj_memory_range(oop obj) const; public: diff --git a/src/hotspot/share/gc/z/zCollectedHeap.cpp b/src/hotspot/share/gc/z/zCollectedHeap.cpp index f60fa38da13..78dcbbc29c4 100644 --- a/src/hotspot/share/gc/z/zCollectedHeap.cpp +++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp @@ -161,11 +161,7 @@ HeapWord* ZCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_si } oop ZCollectedHeap::array_allocate(Klass* klass, size_t size, int length, bool do_zero, TRAPS) { - if (!do_zero) { - return CollectedHeap::array_allocate(klass, size, length, false /* do_zero */, THREAD); - } - - ZObjArrayAllocator allocator(klass, size, length, THREAD); + ZObjArrayAllocator allocator(klass, size, length, do_zero, THREAD); return allocator.allocate(); } diff --git a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp index 69cfc58bc03..391ba264a74 100644 --- a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp +++ b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp @@ -27,36 +27,64 @@ #include "gc/z/zUtils.inline.hpp" #include "oops/arrayKlass.hpp" #include "runtime/interfaceSupport.inline.hpp" +#include "utilities/debug.hpp" -ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, Thread* thread) : - ObjArrayAllocator(klass, word_size, length, false /* do_zero */, thread) {} +ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread) : + ObjArrayAllocator(klass, word_size, length, do_zero, thread) {} -oop ZObjArrayAllocator::finish(HeapWord* mem) const { - // Initialize object header and length field - ObjArrayAllocator::finish(mem); +void ZObjArrayAllocator::yield_for_safepoint() const { + ThreadBlockInVM tbivm(JavaThread::cast(_thread)); +} - // Keep the array alive across safepoints through an invisible - // root. Invisible roots are not visited by the heap itarator - // and the marking logic will not attempt to follow its elements. - ZThreadLocalData::set_invisible_root(_thread, (oop*)&mem); +oop ZObjArrayAllocator::initialize(HeapWord* mem) const { + // ZGC specializes the initialization by performing segmented clearing + // to allow shorter time-to-safepoints. + + if (!_do_zero) { + // No need for ZGC specialization + return ObjArrayAllocator::initialize(mem); + } // A max segment size of 64K was chosen because microbenchmarking // suggested that it offered a good trade-off between allocation // time and time-to-safepoint const size_t segment_max = ZUtils::bytes_to_words(64 * K); - const size_t skip = arrayOopDesc::header_size(ArrayKlass::cast(_klass)->element_type()); - size_t remaining = _word_size - skip; + const BasicType element_type = ArrayKlass::cast(_klass)->element_type(); + const size_t header = arrayOopDesc::header_size(element_type); + const size_t payload_size = _word_size - header; + + if (payload_size <= segment_max) { + // To small to use segmented clearing + return ObjArrayAllocator::initialize(mem); + } + + // Segmented clearing + + // The array is going to be exposed before it has been completely + // cleared, therefore we can't expose the header at the end of this + // function. Instead explicitly initialize it according to our needs. + arrayOopDesc::set_mark(mem, markWord::prototype()); + arrayOopDesc::release_set_klass(mem, _klass); + assert(_length >= 0, "length should be non-negative"); + arrayOopDesc::set_length(mem, _length); + + // Keep the array alive across safepoints through an invisible + // root. Invisible roots are not visited by the heap itarator + // and the marking logic will not attempt to follow its elements. + // Relocation knows how to dodge iterating over such objects. + ZThreadLocalData::set_invisible_root(_thread, (oop*)&mem); + + for (size_t processed = 0; processed < payload_size; processed += segment_max) { + // Calculate segment + HeapWord* const start = (HeapWord*)(mem + header + processed); + const size_t remaining = payload_size - processed; + const size_t segment_size = MIN2(remaining, segment_max); - while (remaining > 0) { // Clear segment - const size_t segment = MIN2(remaining, segment_max); - Copy::zero_to_words(mem + (_word_size - remaining), segment); - remaining -= segment; + Copy::zero_to_words(start, segment_size); - if (remaining > 0) { - // Safepoint - ThreadBlockInVM tbivm(JavaThread::cast(_thread)); - } + // Safepoint + yield_for_safepoint(); } ZThreadLocalData::clear_invisible_root(_thread); diff --git a/src/hotspot/share/gc/z/zObjArrayAllocator.hpp b/src/hotspot/share/gc/z/zObjArrayAllocator.hpp index a0d0913abb8..009e0f0da90 100644 --- a/src/hotspot/share/gc/z/zObjArrayAllocator.hpp +++ b/src/hotspot/share/gc/z/zObjArrayAllocator.hpp @@ -27,10 +27,13 @@ #include "gc/shared/memAllocator.hpp" class ZObjArrayAllocator : public ObjArrayAllocator { -public: - ZObjArrayAllocator(Klass* klass, size_t word_size, int length, Thread* thread); +private: + virtual oop initialize(HeapWord* mem) const override; - virtual oop finish(HeapWord* mem) const; + void yield_for_safepoint() const; + +public: + ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread); }; #endif // SHARE_GC_Z_ZOBJARRAYALLOCATOR_HPP