8231499: Shenandoah: compiler/arraycopy/TestDefaultMethodArrayCloneDeoptC2 fails

Reviewed-by: shade, roland

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

@@ -38,6 +38,7 @@
 #include "opto/movenode.hpp"
 #include "opto/narrowptrnode.hpp"
 #include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
 
 ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
   return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
@@ -461,12 +462,9 @@ const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
 }
 
 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
-  const Type **fields = TypeTuple::fields(4);
-  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // src oop
-  fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL;  // src
-  fields[TypeFunc::Parms+2] = TypeRawPtr::NOTNULL;  // dst
-  fields[TypeFunc::Parms+3] = TypeInt::INT;         // length
-  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+4, fields);
+  const Type **fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeOopPtr::NOTNULL; // src oop
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
 
   // create result type (range)
   fields = TypeTuple::fields(0);
@@ -797,8 +795,6 @@ bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
   return false;
 }
 
-#define XTOP LP64_ONLY(COMMA phase->top())
-
 void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
   Node* ctrl = ac->in(TypeFunc::Control);
   Node* mem = ac->in(TypeFunc::Memory);
@@ -812,13 +808,62 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo
   assert (dest->is_AddP(), "for clone the dst should be the interior ptr");
 
   if (ShenandoahCloneBarrier && clone_needs_barrier(src, phase->igvn())) {
-    Node* call = phase->make_leaf_call(ctrl, mem,
+    // Check if heap is has forwarded objects. If it does, we need to call into the special
+    // routine that would fix up source references before we can continue.
+
+    enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT };
+    Node* region = new RegionNode(PATH_LIMIT);
+    Node* mem_phi = new PhiNode(region, Type::MEMORY, TypeRawPtr::BOTTOM);
+
+    Node* thread = phase->transform_later(new ThreadLocalNode());
+    Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+    Node* gc_state_addr = phase->transform_later(new AddPNode(phase->C->top(), thread, offset));
+
+    uint gc_state_idx = Compile::AliasIdxRaw;
+    const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument
+    debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
+
+    Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
+    Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
+    Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
+    Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
+
+    IfNode* stable_iff  = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
+    Node* stable_ctrl   = phase->transform_later(new IfFalseNode(stable_iff));
+    Node* unstable_ctrl = phase->transform_later(new IfTrueNode(stable_iff));
+
+    // Heap is stable, no need to do anything additional
+    region->init_req(_heap_stable, stable_ctrl);
+    mem_phi->init_req(_heap_stable, mem);
+
+    // Heap is unstable, call into clone barrier stub
+    Node* call = phase->make_leaf_call(unstable_ctrl, mem,
                     ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type(),
                     CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier),
                     "shenandoah_clone",
                     TypeRawPtr::BOTTOM,
-                    src->in(AddPNode::Base), src, dest, length);
+                    src->in(AddPNode::Base));
     call = phase->transform_later(call);
+
+    ctrl = phase->transform_later(new ProjNode(call, TypeFunc::Control));
+    mem = phase->transform_later(new ProjNode(call, TypeFunc::Memory));
+    region->init_req(_heap_unstable, ctrl);
+    mem_phi->init_req(_heap_unstable, mem);
+
+    // Wire up the actual arraycopy stub now
+    ctrl = phase->transform_later(region);
+    mem = phase->transform_later(mem_phi);
+
+    const char* name = "arraycopy";
+    call = phase->make_leaf_call(ctrl, mem,
+                                 OptoRuntime::fast_arraycopy_Type(),
+                                 phase->basictype2arraycopy(T_LONG, NULL, NULL, true, name, true),
+                                 name, TypeRawPtr::BOTTOM,
+                                 src, dest, length
+                                 LP64_ONLY(COMMA phase->top()));
+    call = phase->transform_later(call);
+
+    // Hook up the whole thing into the graph
     phase->igvn().replace_node(ac, call);
   } else {
     BarrierSetC2::clone_at_expansion(phase, ac);

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -286,6 +286,8 @@ oop ShenandoahBarrierSet::oop_load_from_native_barrier(oop obj) {
 }
 
 void ShenandoahBarrierSet::clone_barrier_runtime(oop src) {
-  clone_barrier(src);
+  if (_heap->has_forwarded_objects()) {
+    clone_barrier(src);
+  }
 }
 

src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp

@@ -73,7 +73,7 @@ public:
 
 void ShenandoahBarrierSet::clone_barrier(oop obj) {
   assert(ShenandoahCloneBarrier, "only get here with clone barriers enabled");
-  if (!_heap->has_forwarded_objects()) return;
+  assert(_heap->has_forwarded_objects(), "only when heap is unstable");
 
   // This is called for cloning an object (see jvm.cpp) after the clone
   // has been made. We are not interested in any 'previous value' because

src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp

@@ -76,13 +76,10 @@ JRT_END
 
 // Shenandoah clone barrier: makes sure that references point to to-space
 // in cloned objects.
-JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* src, void* src_ptr, void* dst_ptr, size_t length))
+JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* src))
   oop s = oop(src);
   shenandoah_assert_correct(NULL, s);
   ShenandoahBarrierSet::barrier_set()->clone_barrier(s);
-  Copy::conjoint_jlongs_atomic(reinterpret_cast<jlong*>(src_ptr),
-                               reinterpret_cast<jlong*>(dst_ptr),
-                               length);
 JRT_END
 
 JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_native(oopDesc * src))

src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp

@@ -44,7 +44,7 @@ public:
 
   static oopDesc* load_reference_barrier_native(oopDesc* src);
 
-  static void shenandoah_clone_barrier(oopDesc* src, void* src_ptr, void* dst_ptr, size_t length);
+  static void shenandoah_clone_barrier(oopDesc* src);
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHRUNTIME_HPP