8294900: Refactor ZObjArrayAllocator

Reviewed-by: eosterlund

src/hotspot/share/gc/shared/memAllocator.hpp

@@ -56,12 +56,16 @@ protected:
       _word_size(word_size)
   { }
 
-  // This function clears the memory of the object
+  // Initialization provided by subclasses.
+  virtual oop initialize(HeapWord* mem) const = 0;
+
+  // This function clears the memory of the object.
   void mem_clear(HeapWord* mem) const;
+
   // This finish constructing an oop by installing the mark word and the Klass* pointer
   // last. At the point when the Klass pointer is initialized, this is a constructed object
   // that must be parseable as an oop by concurrent collectors.
-  virtual oop finish(HeapWord* mem) const;
+  oop finish(HeapWord* mem) const;
 
   // Raw memory allocation. This will try to do a TLAB allocation, and otherwise fall
   // back to calling CollectedHeap::mem_allocate().
@@ -72,9 +76,9 @@ protected:
   }
 
 public:
+  // Allocate and fully construct the object, and perform various instrumentation. Could safepoint.
   oop allocate() const;
   oop try_allocate_in_existing_tlab();
-  virtual oop initialize(HeapWord* mem) const = 0;
 };
 
 class ObjAllocator: public MemAllocator {
@@ -85,9 +89,10 @@ public:
 };
 
 class ObjArrayAllocator: public MemAllocator {
+protected:
   const int  _length;
   const bool _do_zero;
-protected:
+
   virtual MemRegion obj_memory_range(oop obj) const;
 
 public:

src/hotspot/share/gc/z/zCollectedHeap.cpp

@@ -161,11 +161,7 @@ HeapWord* ZCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_si
 }
 
 oop ZCollectedHeap::array_allocate(Klass* klass, size_t size, int length, bool do_zero, TRAPS) {
-  if (!do_zero) {
-    return CollectedHeap::array_allocate(klass, size, length, false /* do_zero */, THREAD);
-  }
-
-  ZObjArrayAllocator allocator(klass, size, length, THREAD);
+  ZObjArrayAllocator allocator(klass, size, length, do_zero, THREAD);
   return allocator.allocate();
 }
 

src/hotspot/share/gc/z/zObjArrayAllocator.cpp

@@ -27,36 +27,64 @@
 #include "gc/z/zUtils.inline.hpp"
 #include "oops/arrayKlass.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
+#include "utilities/debug.hpp"
 
-ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, Thread* thread) :
-    ObjArrayAllocator(klass, word_size, length, false /* do_zero */, thread) {}
+ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread) :
+    ObjArrayAllocator(klass, word_size, length, do_zero, thread) {}
 
-oop ZObjArrayAllocator::finish(HeapWord* mem) const {
-  // Initialize object header and length field
-  ObjArrayAllocator::finish(mem);
+void ZObjArrayAllocator::yield_for_safepoint() const {
+  ThreadBlockInVM tbivm(JavaThread::cast(_thread));
+}
 
-  // Keep the array alive across safepoints through an invisible
-  // root. Invisible roots are not visited by the heap itarator
-  // and the marking logic will not attempt to follow its elements.
-  ZThreadLocalData::set_invisible_root(_thread, (oop*)&mem);
+oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
+  // ZGC specializes the initialization by performing segmented clearing
+  // to allow shorter time-to-safepoints.
+
+  if (!_do_zero) {
+    // No need for ZGC specialization
+    return ObjArrayAllocator::initialize(mem);
+  }
 
   // A max segment size of 64K was chosen because microbenchmarking
   // suggested that it offered a good trade-off between allocation
   // time and time-to-safepoint
   const size_t segment_max = ZUtils::bytes_to_words(64 * K);
-  const size_t skip = arrayOopDesc::header_size(ArrayKlass::cast(_klass)->element_type());
-  size_t remaining = _word_size - skip;
+  const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
+  const size_t header = arrayOopDesc::header_size(element_type);
+  const size_t payload_size = _word_size - header;
+
+  if (payload_size <= segment_max) {
+    // To small to use segmented clearing
+    return ObjArrayAllocator::initialize(mem);
+  }
+
+  // Segmented clearing
+
+  // The array is going to be exposed before it has been completely
+  // cleared, therefore we can't expose the header at the end of this
+  // function. Instead explicitly initialize it according to our needs.
+  arrayOopDesc::set_mark(mem, markWord::prototype());
+  arrayOopDesc::release_set_klass(mem, _klass);
+  assert(_length >= 0, "length should be non-negative");
+  arrayOopDesc::set_length(mem, _length);
+
+  // Keep the array alive across safepoints through an invisible
+  // root. Invisible roots are not visited by the heap itarator
+  // and the marking logic will not attempt to follow its elements.
+  // Relocation knows how to dodge iterating over such objects.
+  ZThreadLocalData::set_invisible_root(_thread, (oop*)&mem);
+
+  for (size_t processed = 0; processed < payload_size; processed += segment_max) {
+    // Calculate segment
+    HeapWord* const start = (HeapWord*)(mem + header + processed);
+    const size_t remaining = payload_size - processed;
+    const size_t segment_size = MIN2(remaining, segment_max);
 
-  while (remaining > 0) {
     // Clear segment
-    const size_t segment = MIN2(remaining, segment_max);
-    Copy::zero_to_words(mem + (_word_size - remaining), segment);
-    remaining -= segment;
+    Copy::zero_to_words(start, segment_size);
 
-    if (remaining > 0) {
-      // Safepoint
-      ThreadBlockInVM tbivm(JavaThread::cast(_thread));
-    }
+    // Safepoint
+    yield_for_safepoint();
   }
 
   ZThreadLocalData::clear_invisible_root(_thread);

src/hotspot/share/gc/z/zObjArrayAllocator.hpp

@@ -27,10 +27,13 @@
 #include "gc/shared/memAllocator.hpp"
 
 class ZObjArrayAllocator : public ObjArrayAllocator {
-public:
-  ZObjArrayAllocator(Klass* klass, size_t word_size, int length, Thread* thread);
+private:
+  virtual oop initialize(HeapWord* mem) const override;
 
-  virtual oop finish(HeapWord* mem) const;
+  void yield_for_safepoint() const;
+
+public:
+  ZObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread);
 };
 
 #endif // SHARE_GC_Z_ZOBJARRAYALLOCATOR_HPP