8226695: Shenandoah: Wire C1 and C2 IN_NATIVE barrier

Reviewed-by: roland

src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp

@@ -27,6 +27,7 @@
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
 
@@ -183,7 +184,21 @@ void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result)
     return;
   }
 
-  LIRGenerator *gen = access.gen();
+  LIRGenerator* gen = access.gen();
+
+  DecoratorSet decorators = access.decorators();
+  if ((decorators & IN_NATIVE) != 0) {
+    assert(access.is_oop(), "IN_NATIVE access only for oop values");
+    BarrierSetC1::load_at_resolved(access, result);
+    LIR_OprList* args = new LIR_OprList();
+    args->append(result);
+    BasicTypeList signature;
+    signature.append(T_OBJECT);
+    LIR_Opr call_result = gen->call_runtime(&signature, args, CAST_FROM_FN_PTR(address, ShenandoahRuntime::oop_load_from_native_barrier),
+                                            objectType, NULL);
+    __ move(call_result, result);
+    return;
+  }
 
   if (ShenandoahLoadRefBarrier) {
     LIR_Opr tmp = gen->new_register(T_OBJECT);
@@ -195,7 +210,6 @@ void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result)
   }
 
   if (ShenandoahKeepAliveBarrier) {
-    DecoratorSet decorators = access.decorators();
     bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0;
     bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
     bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
@@ -229,3 +243,10 @@ void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob)
                                                               "shenandoah_pre_barrier_slow",
                                                               false, &pre_code_gen_cl);
 }
+
+const char* ShenandoahBarrierSetC1::rtcall_name_for_address(address entry) {
+  if (entry == CAST_FROM_FN_PTR(address, ShenandoahRuntime::oop_load_from_native_barrier)) {
+    return "ShenandoahRuntime::oop_load_from_native_barrier";
+  }
+  return NULL;
+}

src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp

@@ -203,6 +203,7 @@ protected:
 public:
 
   virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob);
+  virtual const char* rtcall_name_for_address(address entry);
 };
 
 #endif // SHARE_GC_SHENANDOAH_C1_SHENANDOAHBARRIERSETC1_HPP

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

@@ -472,6 +472,19 @@ const TypeFunc* ShenandoahBarrierSetC2::shenandoah_clone_barrier_Type() {
   return TypeFunc::make(domain, range);
 }
 
+const TypeFunc* ShenandoahBarrierSetC2::oop_load_from_native_barrier_Type() {
+  const Type **fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  // create result type (range)
+  fields = TypeTuple::fields(1);
+  fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
 const TypeFunc* ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type() {
   const Type **fields = TypeTuple::fields(1);
   fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
@@ -542,6 +555,19 @@ Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val
   Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
   Node* load = BarrierSetC2::load_at_resolved(access, val_type);
 
+  if ((decorators & IN_NATIVE) != 0) {
+    assert(access.is_oop(), "IN_NATIVE access only for oop values");
+    assert(access.is_parse_access(), "IN_NATIVE access only during parsing");
+    GraphKit* kit = static_cast<C2ParseAccess &>(access).kit();
+    Node* call = kit->make_runtime_call(GraphKit::RC_LEAF,
+                                        oop_load_from_native_barrier_Type(),
+                                        CAST_FROM_FN_PTR(address, ShenandoahRuntime::oop_load_from_native_barrier),
+                                        "ShenandoahRuntime::oop_load_from_native_barrier",
+                                        NULL, load);
+    Node* proj = kit->gvn().transform(new ProjNode(call, TypeFunc::Parms+0));
+    return kit->gvn().transform(new CheckCastPPNode(kit->control(), proj, load->bottom_type()));
+  }
+
   if (access.is_oop()) {
     if (ShenandoahLoadRefBarrier) {
       load = new ShenandoahLoadReferenceBarrierNode(NULL, load);
@@ -723,7 +749,8 @@ bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
 
   return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
          strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
-         strcmp(call->_name, "shenandoah_wb_pre") == 0;
+         strcmp(call->_name, "shenandoah_wb_pre") == 0 ||
+         strcmp(call->_name, "ShenandoahRuntime::oop_load_from_native_barrier") == 0;
 }
 
 Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
@@ -1155,6 +1182,11 @@ bool ShenandoahBarrierSetC2::escape_add_to_con_graph(ConnectionGraph* conn_graph
     case Op_ShenandoahLoadReferenceBarrier:
       conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), delayed_worklist);
       return true;
+    case Op_CallLeaf:
+      if (strcmp(n->as_CallLeaf()->_name, "ShenandoahRuntime::oop_load_from_native_barrier") == 0) {
+        conn_graph->map_ideal_node(n, conn_graph->phantom_obj);
+        return true;
+      }
     default:
       // Nothing
       break;

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp

@@ -103,6 +103,7 @@ public:
   static const TypeFunc* write_ref_field_pre_entry_Type();
   static const TypeFunc* shenandoah_clone_barrier_Type();
   static const TypeFunc* shenandoah_load_reference_barrier_Type();
+  static const TypeFunc* oop_load_from_native_barrier_Type();
   virtual bool has_load_barriers() const { return true; }
 
   // This is the entry-point for the backend to perform accesses through the Access API.

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -358,3 +358,7 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
     }
   }
 }
+
+oop ShenandoahBarrierSet::oop_load_from_native_barrier(oop obj) {
+  return load_reference_barrier(obj);
+}

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp

@@ -83,6 +83,8 @@ public:
   void write_ref_field_work(void* v, oop o, bool release = false);
   void write_region(MemRegion mr);
 
+  oop oop_load_from_native_barrier(oop obj);
+
   virtual void on_thread_create(Thread* thread);
   virtual void on_thread_destroy(Thread* thread);
   virtual void on_thread_attach(Thread* thread);

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -65,7 +65,7 @@ template <DecoratorSet decorators, typename BarrierSetT>
 template <typename T>
 inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
   oop value = Raw::oop_load_not_in_heap(addr);
-  value = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(value);
+  value = ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(value);
   keep_alive_if_weak(decorators, value);
   return value;
 }

src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp

@@ -65,3 +65,7 @@ JRT_END
 JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* obj))
   ShenandoahBarrierSet::barrier_set()->write_region(MemRegion((HeapWord*) obj, obj->size()));
 JRT_END
+
+JRT_LEAF(oopDesc*, ShenandoahRuntime::oop_load_from_native_barrier(oopDesc* src))
+  return (oopDesc*) ShenandoahBarrierSet::barrier_set()->oop_load_from_native_barrier(oop(src));
+JRT_END

src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp

@@ -39,6 +39,8 @@ public:
 
   static oopDesc* load_reference_barrier_JRT(oopDesc* src);
 
+  static oopDesc* oop_load_from_native_barrier(oopDesc* src);
+
   static void shenandoah_clone_barrier(oopDesc* obj);
 };