8242130: Shenandoah: Simplify arraycopy-barrier dispatching

Reviewed-by: shade

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -56,7 +56,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
       // Avoid calling runtime if count == 0
       __ cbz(count, done);
 
-      // Is marking active?
+      // Is GC active?
       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
       __ ldrb(rscratch1, gc_state);
       if (dest_uninitialized) {
@@ -69,17 +69,9 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
 
       __ push(saved_regs, sp);
       if (UseCompressedOops) {
-        if (dest_uninitialized) {
-          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
-        } else {
-          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
-        }
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry), src, dst, count);
       } else {
-        if (dest_uninitialized) {
-          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
-        } else {
-          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
-        }
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry), src, dst, count);
       }
       __ pop(saved_regs, sp);
       __ bind(done);

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

@@ -77,7 +77,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
       __ testptr(count, count);
       __ jcc(Assembler::zero, done);
 
-      // Avoid runtime call when not marking.
+      // Avoid runtime call when not active.
       Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
       int flags = ShenandoahHeap::HAS_FORWARDED;
       if (!dest_uninitialized) {
@@ -87,25 +87,21 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
       __ jcc(Assembler::zero, done);
 
       __ pusha();                      // push registers
+
 #ifdef _LP64
       assert(src == rdi, "expected");
       assert(dst == rsi, "expected");
       assert(count == rdx, "expected");
       if (UseCompressedOops) {
-        if (dest_uninitialized) {
-          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
-        } else {
-          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
-        }
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry),
+                        src, dst, count);
       } else
 #endif
       {
-        if (dest_uninitialized) {
-          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
-        } else {
-          __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
-        }
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop_entry),
+                        src, dst, count);
       }
+
       __ popa();
       __ bind(done);
       NOT_LP64(__ pop(thread);)

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp

@@ -68,13 +68,8 @@ public:
 
   bool is_aligned(HeapWord* hw);
 
-  template <class T> void
-  write_ref_array_pre_work(T* src, T* dst, size_t count, bool dest_uninitialized);
-
-  inline void arraycopy_pre(oop* src, oop* dst, size_t count);
-  inline void arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count);
-  inline void arraycopy_update(oop* src, size_t count);
-  inline void arraycopy_update(narrowOop* src, size_t count);
+  template <class T>
+  inline void arraycopy_barrier(T* src, T* dst, size_t count);
   inline void clone_barrier(oop src);
   void clone_barrier_runtime(oop src);
 
@@ -112,18 +107,21 @@ public:
 
 private:
   template <class T>
-  inline void arraycopy_pre_work(T* src, T* dst, size_t count);
+  inline void arraycopy_marking(T* ary, size_t count);
+  template <class T>
+  inline void arraycopy_evacuation(T* src, size_t count);
+  template <class T>
+  inline void arraycopy_update(T* src, size_t count);
+
   template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
   inline void arraycopy_work(T* src, size_t count);
-  template <class T>
-  inline void arraycopy_update_impl(T* src, size_t count);
 
   oop load_reference_barrier_impl(oop obj);
 
   template <class T>
   oop load_reference_barrier_native_impl(oop obj, T* load_addr);
 
-  inline bool skip_bulk_update(HeapWord* dst);
+  inline bool need_bulk_update(HeapWord* dst);
 public:
   // Callbacks for runtime accesses.
   template <DecoratorSet decorators, typename BarrierSetT = ShenandoahBarrierSet>

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -249,9 +249,9 @@ bool ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy
                                                                                          arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
                                                                                          size_t length) {
   ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
-  bs->arraycopy_pre(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw),
-                    arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw),
-                    length);
+  bs->arraycopy_barrier(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw),
+                        arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw),
+                        length);
   return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length);
 }
 
@@ -285,46 +285,47 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
 }
 
 template <class T>
-void ShenandoahBarrierSet::arraycopy_pre_work(T* src, T* dst, size_t count) {
-  if (_heap->is_concurrent_mark_in_progress() &&
-      !_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(dst))) {
-    arraycopy_work<T, false, false, true>(dst, count);
+void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
+  if (count == 0) {
+    return;
   }
-
-  if (_heap->has_forwarded_objects()) {
-    arraycopy_update_impl(src, count);
+  int gc_state = _heap->gc_state();
+  if ((gc_state & ShenandoahHeap::MARKING) != 0) {
+    arraycopy_marking(dst, count);
+  } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
+    arraycopy_evacuation(src, count);
+  } else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) {
+    arraycopy_update(src, count);
   }
 }
 
-void ShenandoahBarrierSet::arraycopy_pre(oop* src, oop* dst, size_t count) {
-  arraycopy_pre_work(src, dst, count);
-}
-
-void ShenandoahBarrierSet::arraycopy_pre(narrowOop* src, narrowOop* dst, size_t count) {
-  arraycopy_pre_work(src, dst, count);
-}
-
-inline bool ShenandoahBarrierSet::skip_bulk_update(HeapWord* dst) {
-  return dst >= _heap->heap_region_containing(dst)->get_update_watermark();
-}
-
 template <class T>
-void ShenandoahBarrierSet::arraycopy_update_impl(T* src, size_t count) {
-  if (skip_bulk_update(reinterpret_cast<HeapWord*>(src))) return;
-  if (_heap->is_evacuation_in_progress()) {
+void ShenandoahBarrierSet::arraycopy_marking(T* array, size_t count) {
+  assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
+  if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(array))) {
+    arraycopy_work<T, false, false, true>(array, count);
+  }
+}
+
+inline bool ShenandoahBarrierSet::need_bulk_update(HeapWord* ary) {
+  return ary < _heap->heap_region_containing(ary)->get_update_watermark();
+}
+
+template <class T>
+void ShenandoahBarrierSet::arraycopy_evacuation(T* src, size_t count) {
+  assert(_heap->is_evacuation_in_progress(), "only during evacuation");
+  if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) {
     ShenandoahEvacOOMScope oom_evac;
     arraycopy_work<T, true, true, false>(src, count);
-  } else if (_heap->has_forwarded_objects()) {
+  }
+}
+
+template <class T>
+void ShenandoahBarrierSet::arraycopy_update(T* src, size_t count) {
+  assert(_heap->is_update_refs_in_progress(), "only during update-refs");
+  if (need_bulk_update(reinterpret_cast<HeapWord*>(src))) {
     arraycopy_work<T, true, false, false>(src, count);
   }
 }
 
-void ShenandoahBarrierSet::arraycopy_update(oop* src, size_t count) {
-  arraycopy_update_impl(src, count);
-}
-
-void ShenandoahBarrierSet::arraycopy_update(narrowOop* src, size_t count) {
-  arraycopy_update_impl(src, count);
-}
-
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP

src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp

@@ -82,7 +82,7 @@ void ShenandoahBarrierSet::clone_barrier(oop obj) {
   // that potentially need to be updated.
 
   shenandoah_assert_correct(NULL, obj);
-  if (skip_bulk_update(cast_from_oop<HeapWord*>(obj))) return;
+  if (!need_bulk_update(cast_from_oop<HeapWord*>(obj))) return;
   if (_heap->is_evacuation_in_progress()) {
     ShenandoahEvacOOMScope evac_scope;
     ShenandoahUpdateRefsForOopClosure</* evac = */ true, /* enqueue */ false> cl;

src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp

@@ -31,24 +31,14 @@
 #include "oops/oop.inline.hpp"
 #include "utilities/copy.hpp"
 
-void ShenandoahRuntime::write_ref_array_pre_oop_entry(oop* src, oop* dst, size_t length) {
+void ShenandoahRuntime::arraycopy_barrier_oop_entry(oop* src, oop* dst, size_t length) {
   ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set();
-  bs->arraycopy_pre(src, dst, length);
+  bs->arraycopy_barrier(src, dst, length);
 }
 
-void ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length) {
+void ShenandoahRuntime::arraycopy_barrier_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length) {
   ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set();
-  bs->arraycopy_pre(src, dst, length);
-}
-
-void ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry(oop* src, oop* dst, size_t length) {
-  ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set();
-  bs->arraycopy_update(src, length);
-}
-
-void ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length) {
-  ShenandoahBarrierSet *bs = ShenandoahBarrierSet::barrier_set();
-  bs->arraycopy_update(src, length);
+  bs->arraycopy_barrier(src, dst, length);
 }
 
 // Shenandoah pre write barrier slowpath

src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp

@@ -33,10 +33,9 @@ class oopDesc;
 
 class ShenandoahRuntime : public AllStatic {
 public:
-  static void write_ref_array_pre_oop_entry(oop* src, oop* dst, size_t length);
-  static void write_ref_array_pre_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length);
-  static void write_ref_array_pre_duinit_oop_entry(oop* src, oop* dst, size_t length);
-  static void write_ref_array_pre_duinit_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length);
+  static void arraycopy_barrier_oop_entry(oop* src, oop* dst, size_t length);
+  static void arraycopy_barrier_narrow_oop_entry(narrowOop* src, narrowOop* dst, size_t length);
+
   static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread);
 
   static oopDesc* load_reference_barrier(oopDesc* src, oop* load_addr);