8294729: [s390] Implement nmethod entry barriers

Reviewed-by: mdoerr, eosterlund

src/hotspot/cpu/s390/assembler_s390.hpp

@@ -3121,6 +3121,9 @@ class Assembler : public AbstractAssembler {
   static bool is_z_algr(long x) {
     return (ALGR_ZOPC == (x & RRE_MASK));
   }
+  static bool is_z_cfi(long x) {
+    return (CFI_ZOPC == (x & RIL_MASK));
+  }
   static bool is_z_lb(long x) {
     return (LB_ZOPC == (x & LB_MASK));
   }

src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp

@@ -27,6 +27,8 @@
 #include "asm/macroAssembler.inline.hpp"
 #include "c1/c1_MacroAssembler.hpp"
 #include "c1/c1_Runtime1.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/tlab_globals.hpp"
 #include "interpreter/interpreter.hpp"
@@ -72,6 +74,9 @@ void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_by
   generate_stack_overflow_check(bang_size_in_bytes);
   save_return_pc();
   push_frame(frame_size_in_bytes);
+
+  BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  bs->nmethod_entry_barrier(this);
 }
 
 void C1_MacroAssembler::verified_entry(bool breakAtEntry) {

src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -25,10 +25,13 @@
 
 #include "precompiled.hpp"
 #include "asm/macroAssembler.inline.hpp"
+#include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetAssembler.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
 #include "interpreter/interp_masm.hpp"
 #include "oops/compressedOops.hpp"
 #include "runtime/jniHandles.hpp"
+#include "runtime/stubRoutines.hpp"
 #include "utilities/macros.hpp"
 
 #define __ masm->
@@ -119,3 +122,28 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re
   __ z_nill(obj, ~JNIHandles::weak_tag_mask);
   __ z_lg(obj, 0, obj); // Resolve (untagged) jobject.
 }
+
+void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
+  BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
+  if (bs_nm == nullptr) {
+    return;
+  }
+
+  __ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");
+
+    // Load jump addr:
+    __ load_const(Z_R1_scratch, (uint64_t)StubRoutines::zarch::nmethod_entry_barrier()); // 2*6 bytes
+
+    // Load value from current java object:
+    __ z_lg(Z_R0_scratch, in_bytes(bs_nm->thread_disarmed_offset()), Z_thread); // 6 bytes
+
+    // Compare to current patched value:
+    __ z_cfi(Z_R0_scratch, /* to be patched */ -1); // 6 bytes (2 + 4 byte imm val)
+
+    // Conditional Jump
+    __ z_larl(Z_R14, (Assembler::instr_len((unsigned long)LARL_ZOPC) + Assembler::instr_len((unsigned long)BCR_ZOPC)) / 2); // 6 bytes
+    __ z_bcr(Assembler::bcondNotEqual, Z_R1_scratch); // 2 bytes
+
+    // Fall through to method body.
+  __ block_comment("} nmethod_entry_barrier (nmethod_entry_barrier)");
+}

src/hotspot/cpu/s390/gc/shared/barrierSetAssembler_s390.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -49,6 +49,8 @@ public:
   virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
                                              Register obj, Register tmp, Label& slowpath);
 
+  virtual void nmethod_entry_barrier(MacroAssembler* masm);
+
   virtual void barrier_stubs_init() {}
 };
 

src/hotspot/cpu/s390/gc/shared/barrierSetNMethod_s390.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,18 +23,107 @@
  */
 
 #include "precompiled.hpp"
+#include "asm/assembler.inline.hpp"
+#include "code/codeBlob.hpp"
+#include "code/nativeInst.hpp"
+#include "code/nmethod.hpp"
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "utilities/debug.hpp"
 
+class NativeMethodBarrier: public NativeInstruction {
+  private:
+    static const int PATCHABLE_INSTRUCTION_OFFSET = 3*6; // bytes
+
+    address get_barrier_start_address() const {
+      return NativeInstruction::addr_at(0);
+    }
+
+    address get_patchable_data_address() const {
+      address inst_addr = get_barrier_start_address() + PATCHABLE_INSTRUCTION_OFFSET;
+
+      debug_only(Assembler::is_z_cfi(*((long*)inst_addr)));
+      return inst_addr + 2;
+    }
+
+  public:
+    static const int BARRIER_TOTAL_LENGTH = PATCHABLE_INSTRUCTION_OFFSET + 2*6 + 2; // bytes
+
+    int get_guard_value() const {
+      address data_addr = get_patchable_data_address();
+      // Return guard instruction value
+      return *((int32_t*)data_addr);
+    }
+
+    void set_guard_value(int value) {
+      int32_t* data_addr = (int32_t*)get_patchable_data_address();
+
+      // Set guard instruction value
+      *data_addr = value;
+    }
+
+    #ifdef ASSERT
+      void verify() const {
+        int offset = 0; // bytes
+        const address start = get_barrier_start_address();
+
+        MacroAssembler::is_load_const(/* address */ start + offset); // two instructions
+        offset += Assembler::instr_len(&start[offset]);
+        offset += Assembler::instr_len(&start[offset]);
+
+        Assembler::is_z_lg(*((long*)(start + offset)));
+        offset += Assembler::instr_len(&start[offset]);
+
+        Assembler::is_z_cfi(*((long*)(start + offset)));
+        offset += Assembler::instr_len(&start[offset]);
+
+        Assembler::is_z_larl(*((long*)(start + offset)));
+        offset += Assembler::instr_len(&start[offset]);
+
+        Assembler::is_z_bcr(*((long*)(start + offset)));
+        offset += Assembler::instr_len(&start[offset]);
+
+        assert(offset == BARRIER_TOTAL_LENGTH, "check offset == barrier length constant");
+      }
+    #endif
+
+};
+
+static NativeMethodBarrier* get_nmethod_barrier(nmethod* nm) {
+  address barrier_address = nm->code_begin() + nm->frame_complete_offset() - NativeMethodBarrier::BARRIER_TOTAL_LENGTH;
+  auto barrier = reinterpret_cast<NativeMethodBarrier*>(barrier_address);
+
+  debug_only(barrier->verify());
+  return barrier;
+}
+
 void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
-  ShouldNotReachHere();
+  // Not required on s390 as a valid backchain is present
+  return;
+}
+
+void BarrierSetNMethod::arm(nmethod* nm, int arm_value) {
+  if (!supports_entry_barrier(nm)) {
+    return;
+  }
+
+  NativeMethodBarrier* barrier = get_nmethod_barrier(nm);
+  barrier->set_guard_value(arm_value);
 }
 
 void BarrierSetNMethod::disarm(nmethod* nm) {
-  ShouldNotReachHere();
+  if (!supports_entry_barrier(nm)) {
+    return;
+  }
+
+  NativeMethodBarrier* barrier = get_nmethod_barrier(nm);
+  barrier->set_guard_value(disarmed_value());
 }
 
 bool BarrierSetNMethod::is_armed(nmethod* nm) {
-  ShouldNotReachHere();
-  return false;
+  if (!supports_entry_barrier(nm)) {
+    return false;
+  }
+
+  NativeMethodBarrier* barrier = get_nmethod_barrier(nm);
+  return barrier->get_guard_value() != disarmed_value();
 }

src/hotspot/cpu/s390/s390.ad

@@ -810,6 +810,8 @@ void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
 
 //=============================================================================
 
+#include "gc/shared/barrierSetAssembler.hpp"
+
 #if !defined(PRODUCT)
 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
   Compile* C = ra_->C;
@@ -837,6 +839,10 @@ void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
   }
   st->print_cr("push_frame %d", (int)-framesize);
   st->print("\t");
+
+  if (C->stub_function() == NULL) {
+    st->print("nmethod entry barrier\n\t");
+  }
 }
 #endif
 
@@ -890,6 +896,13 @@ void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
     ConstantTable& constant_table = C->output()->constant_table();
     constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
   }
+
+  if (C->stub_function() == NULL) {
+    BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
+    bs->nmethod_entry_barrier(&_masm);
+  }
+
+  C->output()->set_frame_complete(cbuf.insts_size());
 }
 
 uint MachPrologNode::size(PhaseRegAlloc *ra_) const {

src/hotspot/cpu/s390/sharedRuntime_s390.cpp

@@ -29,6 +29,7 @@
 #include "code/icBuffer.hpp"
 #include "code/vtableStubs.hpp"
 #include "compiler/oopMap.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
 #include "gc/shared/gcLocker.hpp"
 #include "interpreter/interpreter.hpp"
 #include "interpreter/interp_masm.hpp"
@@ -1541,6 +1542,9 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
                                                           // Just resize the existing one.
 #endif
 
+  BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
+  bs->nmethod_entry_barrier(masm);
+
   wrapper_FrameDone = __ offset();
 
   __ verify_thread();

src/hotspot/cpu/s390/stubGenerator_s390.cpp

@@ -28,6 +28,7 @@
 #include "registerSaver_s390.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetAssembler.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
 #include "interpreter/interpreter.hpp"
 #include "interpreter/interp_masm.hpp"
 #include "memory/universe.hpp"
@@ -2857,6 +2858,47 @@ class StubGenerator: public StubCodeGenerator {
     return start;
   }
 
+  address generate_nmethod_entry_barrier() {
+    __ align(CodeEntryAlignment);
+    StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
+
+    address start = __ pc();
+
+    int nbytes_volatile = (8 + 5) * BytesPerWord;
+
+    // VM-Call Prologue
+    __ save_return_pc();
+    __ push_frame_abi160(nbytes_volatile);
+    __ save_volatile_regs(Z_SP, frame::z_abi_160_size, true, false);
+
+    // Prep arg for VM call
+    // Create ptr to stored return_pc in caller frame.
+    __ z_la(Z_ARG1, _z_abi(return_pc) + frame::z_abi_160_size + nbytes_volatile, Z_R0, Z_SP);
+
+    // VM-Call: BarrierSetNMethod::nmethod_stub_entry_barrier(address* return_address_ptr)
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetNMethod::nmethod_stub_entry_barrier));
+    __ z_ltr(Z_R0_scratch, Z_RET);
+
+    // VM-Call Epilogue
+    __ restore_volatile_regs(Z_SP, frame::z_abi_160_size, true, false);
+    __ pop_frame();
+    __ restore_return_pc();
+
+    // Check return val of VM-Call
+    __ z_bcr(Assembler::bcondZero, Z_R14);
+
+    // Pop frame built in prologue.
+    // Required so wrong_method_stub can deduce caller.
+    __ pop_frame();
+    __ restore_return_pc();
+
+    // VM-Call indicates deoptimization required
+    __ load_const_optimized(Z_R1_scratch, SharedRuntime::get_handle_wrong_method_stub());
+    __ z_br(Z_R1_scratch);
+
+    return start;
+  }
+
   address generate_cont_thaw(bool return_barrier, bool exception) {
     if (!Continuations::enabled()) return nullptr;
     Unimplemented();
@@ -2998,6 +3040,12 @@ class StubGenerator: public StubCodeGenerator {
       StubRoutines::_sha512_implCompressMB = generate_SHA512_stub(true,  "SHA512_multiBlock");
     }
 
+    // nmethod entry barriers for concurrent class unloading
+    BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
+    if (bs_nm != NULL) {
+      StubRoutines::zarch::_nmethod_entry_barrier = generate_nmethod_entry_barrier();
+    }
+
 #ifdef COMPILER2
     if (UseMultiplyToLenIntrinsic) {
       StubRoutines::_multiplyToLen = generate_multiplyToLen();

src/hotspot/cpu/s390/stubRoutines_s390.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2016, 2017 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -38,6 +38,8 @@ address StubRoutines::zarch::_partial_subtype_check = NULL;
 // Comapct string intrinsics: Translate table for string inflate intrinsic. Used by trot instruction.
 address StubRoutines::zarch::_trot_table_addr = NULL;
 
+address StubRoutines::zarch::_nmethod_entry_barrier = NULL;
+
 int StubRoutines::zarch::_atomic_memory_operation_lock = StubRoutines::zarch::unlocked;
 
 #define __ masm->

src/hotspot/cpu/s390/stubRoutines_s390.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2016, 2017 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -78,6 +78,8 @@ class zarch {
   static address _trot_table_addr;
   static jlong   _trot_table[TROT_COLUMN_SIZE];
 
+  static address _nmethod_entry_barrier;
+
  public:
   // Global lock for everyone who needs to use atomic_compare_and_exchange
   // or atomic_increment -- should probably use more locks for more
@@ -98,6 +100,8 @@ class zarch {
 
   // Comapct string intrinsics: Translate table for string inflate intrinsic. Used by trot instruction.
   static void generate_load_trot_table_addr(MacroAssembler* masm, Register table);
+
+  static address nmethod_entry_barrier() { return _nmethod_entry_barrier; }
 };
 
 #endif // CPU_S390_STUBROUTINES_S390_HPP