8232010: Shenandoah: implement self-fixing native barrier

Reviewed-by: rkennke

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

@@ -333,7 +333,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembl
 #endif
 }
 
-void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst) {
+void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Address src) {
   if (!ShenandoahLoadRefBarrier) {
     return;
   }
@@ -341,6 +341,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler
   Label done;
   Label not_null;
   Label slow_path;
+  __ block_comment("load_reference_barrier_native { ");
 
   // null check
   __ testptr(dst, dst);
@@ -371,7 +372,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler
   __ bind(slow_path);
 
   if (dst != rax) {
-    __ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
+    __ push(rax);
   }
   __ push(rcx);
   __ push(rdx);
@@ -388,8 +389,9 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler
   __ push(r15);
 #endif
 
-  __ movptr(rdi, rax);
-  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), rdi);
+  assert_different_registers(dst, rsi);
+  __ lea(rsi, src);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), dst, rsi);
 
 #ifdef _LP64
   __ pop(r15);
@@ -407,10 +409,12 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler
   __ pop(rcx);
 
   if (dst != rax) {
-    __ xchgptr(rax, dst); // Swap back obj with rax.
+    __ movptr(dst, rax);
+    __ pop(rax);
   }
 
   __ bind(done);
+  __ block_comment("load_reference_barrier_native { ");
 }
 
 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
@@ -474,14 +478,43 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
   bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
   bool keep_alive = ((decorators & AS_NO_KEEPALIVE) == 0) || is_traversal_mode;
 
+  Register result_dst = dst;
+  bool use_tmp1_for_dst = false;
+
+  if (on_oop) {
+    // We want to preserve src
+    if (dst == src.base() || dst == src.index()) {
+      // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
+      if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
+        dst = tmp1;
+        use_tmp1_for_dst = true;
+      } else {
+        dst = rdi;
+        __ push(dst);
+      }
+    }
+    assert_different_registers(dst, src.base(), src.index());
+  }
+
   BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+
   if (on_oop) {
     if (not_in_heap && !is_traversal_mode) {
-      load_reference_barrier_native(masm, dst);
+      load_reference_barrier_native(masm, dst, src);
     } else {
       load_reference_barrier(masm, dst);
     }
 
+    if (dst != result_dst) {
+      __ movptr(result_dst, dst);
+
+      if (!use_tmp1_for_dst) {
+        __ pop(dst);
+      }
+
+      dst = result_dst;
+    }
+
     if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
       const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
       assert_different_registers(dst, tmp1, tmp_thread);

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp

@@ -76,7 +76,7 @@ public:
 #endif
 
   void load_reference_barrier(MacroAssembler* masm, Register dst);
-  void load_reference_barrier_native(MacroAssembler* masm, Register dst);
+  void load_reference_barrier_native(MacroAssembler* masm, Register dst, Address src);
 
   void cmpxchg_oop(MacroAssembler* masm,
                    Register res, Address addr, Register oldval, Register newval,

src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp

@@ -215,9 +215,13 @@ void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result)
     assert(access.is_oop(), "IN_NATIVE access only for oop values");
     BarrierSetC1::load_at_resolved(access, result);
     LIR_OprList* args = new LIR_OprList();
+    LIR_Opr addr = access.resolved_addr();
+    addr = ensure_in_register(gen, addr);
     args->append(result);
+    args->append(addr);
     BasicTypeList signature;
     signature.append(T_OBJECT);
+    signature.append(T_ADDRESS);
     LIR_Opr call_result = gen->call_runtime(&signature, args,
                                             CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native),
                                             objectType, NULL);

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -262,7 +262,7 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
   }
 }
 
-oop ShenandoahBarrierSet::oop_load_from_native_barrier(oop obj) {
+oop ShenandoahBarrierSet::oop_load_from_native_barrier(oop obj, oop* load_addr) {
   if (CompressedOops::is_null(obj)) {
     return NULL;
   }
@@ -277,7 +277,13 @@ oop ShenandoahBarrierSet::oop_load_from_native_barrier(oop obj) {
     }
   }
 
-  return load_reference_barrier_not_null(obj);
+  oop fwd = load_reference_barrier_not_null(obj);
+  if (load_addr != NULL && fwd != obj) {
+    // Since we are here and we know the load address, update the reference.
+    ShenandoahHeap::cas_oop(fwd, load_addr, obj);
+  }
+
+  return fwd;
 }
 
 void ShenandoahBarrierSet::clone_barrier_runtime(oop src) {

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp

@@ -84,7 +84,9 @@ public:
 
   void write_ref_field_work(void* v, oop o, bool release = false);
 
-  oop oop_load_from_native_barrier(oop obj);
+  oop oop_load_from_native_barrier(oop obj, oop* load_addr);
+  oop oop_load_from_native_barrier(oop obj, narrowOop* load_addr);
+
 
   virtual void on_thread_create(Thread* thread);
   virtual void on_thread_destroy(Thread* thread);

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -69,7 +69,7 @@ template <DecoratorSet decorators, typename BarrierSetT>
 template <typename T>
 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
   oop value = Raw::oop_load_not_in_heap(addr);
-  value = ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(value);
+  value = ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(value, addr);
   keep_alive_if_weak(decorators, value);
   return value;
 }

src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp

@@ -82,6 +82,6 @@ JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* src))
   ShenandoahBarrierSet::barrier_set()->clone_barrier(s);
 JRT_END
 
-JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_native(oopDesc * src))
-  return (oopDesc*) ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(oop(src));
+JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_native(oopDesc * src, oop* load_addr))
+  return (oopDesc*) ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(oop(src), load_addr);
 JRT_END

src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp

@@ -42,7 +42,7 @@ public:
   static oopDesc* load_reference_barrier_fixup(oopDesc* src, oop* load_addr);
   static oopDesc* load_reference_barrier_fixup_narrow(oopDesc* src, narrowOop* load_addr);
 
-  static oopDesc* load_reference_barrier_native(oopDesc* src);
+  static oopDesc* load_reference_barrier_native(oopDesc* src, oop* load_addr);
 
   static void shenandoah_clone_barrier(oopDesc* src);
 };