8290965: PPC64: Implement post-call NOPs

Reviewed-by: mdoerr

src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp

@@ -2862,6 +2862,7 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest,
   if (info != nullptr) {
     add_call_info_here(info);
   }
+  assert(__ last_calls_return_pc() == __ pc(), "pcn not at return pc");
   __ post_call_nop();
 }
 

src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp

@@ -350,7 +350,7 @@ inline void Thaw<ConfigT>::patch_caller_links(intptr_t* sp, intptr_t* bottom) {
     if (is_entry_frame) {
       callers_sp = _cont.entryFP();
     } else {
-      CodeBlob* cb = CodeCache::find_blob(pc);
+      CodeBlob* cb = CodeCache::find_blob_fast(pc);
       callers_sp = sp + cb->frame_size();
     }
     // set the back link

src/hotspot/cpu/ppc/frame_ppc.cpp

@@ -136,7 +136,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
 
     // It should be safe to construct the sender though it might not be valid.
 
-    frame sender(sender_sp, sender_pc);
+    frame sender(sender_sp, sender_pc, nullptr /* unextended_sp */, nullptr /* fp */, sender_blob);
 
     // Do we have a valid fp?
     address sender_fp = (address) sender.fp();
@@ -196,12 +196,12 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const {
   assert(map->include_argument_oops(), "should be set by clear");
 
   if (jfa->last_Java_pc() != nullptr) {
-    frame fr(jfa->last_Java_sp(), jfa->last_Java_pc());
+    frame fr(jfa->last_Java_sp(), jfa->last_Java_pc(), kind::code_blob);
     return fr;
   }
   // Last_java_pc is not set, if we come here from compiled code. The
   // constructor retrieves the PC from the stack.
-  frame fr(jfa->last_Java_sp());
+  frame fr(jfa->last_Java_sp(), nullptr, kind::code_blob);
   return fr;
 }
 
@@ -229,7 +229,7 @@ frame frame::sender_for_upcall_stub_frame(RegisterMap* map) const {
   assert(jfa->last_Java_sp() > sp(), "must be above this frame on stack");
   map->clear();
   assert(map->include_argument_oops(), "should be set by clear");
-  frame fr(jfa->last_Java_sp(), jfa->last_Java_pc());
+  frame fr(jfa->last_Java_sp(), jfa->last_Java_pc(), kind::code_blob);
 
   return fr;
 }
@@ -451,7 +451,7 @@ intptr_t *frame::initial_deoptimization_info() {
 #ifndef PRODUCT
 // This is a generic constructor which is only used by pns() in debug.cpp.
 // fp is dropped and gets determined by backlink.
-frame::frame(void* sp, void* fp, void* pc) : frame((intptr_t*)sp, (address)pc) {}
+frame::frame(void* sp, void* fp, void* pc) : frame((intptr_t*)sp, (address)pc, kind::unknown) {}
 #endif
 
 BasicObjectLock* frame::interpreter_frame_monitor_end() const {

src/hotspot/cpu/ppc/frame_ppc.hpp

@@ -393,16 +393,26 @@
   inline common_abi* own_abi()     const { return (common_abi*) _sp; }
   inline common_abi* callers_abi() const { return (common_abi*) _fp; }
 
+  enum class kind {
+    unknown,          // The frame's pc is not necessarily in the CodeCache.
+                      // CodeCache::find_blob_fast(void* pc) can yield wrong results in this case and must not be used.
+    code_blob,        // The frame's pc is known to be in the CodeCache but it is likely not in an nmethod.
+                      // CodeCache::find_blob_fast() will be correct but not faster in this case.
+    nmethod           // This is likely the frame of a nmethod.
+                      // The code cache lookup is optimized based on NativePostCallNops.
+  };
+
  private:
 
   // Initialize frame members (_pc and _sp must be given)
-  inline void setup();
+  inline void setup(kind knd);
 
  public:
 
   // Constructors
   inline frame(intptr_t* sp, intptr_t* fp, address pc);
-  inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp = nullptr, intptr_t* fp = nullptr, CodeBlob* cb = nullptr);
+  inline frame(intptr_t* sp, address pc, kind knd = kind::nmethod);
+  inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp, intptr_t* fp = nullptr, CodeBlob* cb = nullptr);
   inline frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map);
   inline frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map, bool on_heap);
 

src/hotspot/cpu/ppc/frame_ppc.inline.hpp

@@ -35,14 +35,14 @@
 // Inline functions for ppc64 frames:
 
 // Initialize frame members (_sp must be given)
-inline void frame::setup() {
+inline void frame::setup(kind knd) {
   if (_pc == nullptr) {
     _pc = (address)own_abi()->lr;
     assert(_pc != nullptr, "must have PC");
   }
 
   if (_cb == nullptr) {
-    _cb = CodeCache::find_blob(_pc);
+    _cb = (knd == kind::nmethod) ? CodeCache::find_blob_fast(_pc) : CodeCache::find_blob(_pc);
   }
 
   if (_unextended_sp == nullptr) {
@@ -89,21 +89,27 @@ inline void frame::setup() {
 inline frame::frame() : _sp(nullptr), _pc(nullptr), _cb(nullptr), _oop_map(nullptr), _deopt_state(unknown),
                         _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(nullptr), _fp(nullptr) {}
 
-inline frame::frame(intptr_t* sp) : frame(sp, nullptr) {}
+inline frame::frame(intptr_t* sp) : frame(sp, nullptr, kind::nmethod) {}
 
 inline frame::frame(intptr_t* sp, intptr_t* fp, address pc) : frame(sp, pc, nullptr, fp, nullptr) {}
 
+inline frame::frame(intptr_t* sp, address pc, kind knd)
+  : _sp(sp), _pc(pc), _cb(nullptr), _oop_map(nullptr),
+    _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(sp), _fp(nullptr) {
+  setup(knd);
+}
+
 inline frame::frame(intptr_t* sp, address pc, intptr_t* unextended_sp, intptr_t* fp, CodeBlob* cb)
   : _sp(sp), _pc(pc), _cb(cb), _oop_map(nullptr),
     _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(unextended_sp), _fp(fp) {
-  setup();
+  setup(kind::nmethod);
 }
 
 inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb, const ImmutableOopMap* oop_map)
   : _sp(sp), _pc(pc), _cb(cb), _oop_map(oop_map),
     _on_heap(false), DEBUG_ONLY(_frame_index(-1) COMMA) _unextended_sp(unextended_sp), _fp(fp) {
   assert(_cb != nullptr, "pc: " INTPTR_FORMAT, p2i(pc));
-  setup();
+  setup(kind::nmethod);
 }
 
 inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address pc, CodeBlob* cb,
@@ -113,7 +119,7 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address
   // In thaw, non-heap frames use this constructor to pass oop_map.  I don't know why.
   assert(_on_heap || _cb != nullptr, "these frames are always heap frames");
   if (cb != nullptr) {
-    setup();
+    setup(kind::nmethod);
   }
 #ifdef ASSERT
   // The following assertion has been disabled because it would sometime trap for Continuation.run,
@@ -300,7 +306,7 @@ inline frame frame::sender_raw(RegisterMap* map) const {
 
   // Must be native-compiled frame, i.e. the marshaling code for native
   // methods that exists in the core system.
-  return frame(sender_sp(), sender_pc());
+  return frame(sender_sp(), sender_pc(), kind::code_blob);
 }
 
 inline frame frame::sender(RegisterMap* map) const {

src/hotspot/cpu/ppc/macroAssembler_ppc.cpp

@@ -1187,8 +1187,12 @@ void MacroAssembler::post_call_nop() {
   if (!Continuations::enabled()) {
     return;
   }
+  // We use CMPI/CMPLI instructions to encode post call nops.
+  // Refer to NativePostCallNop for details.
+  relocate(post_call_nop_Relocation::spec());
   InlineSkippedInstructionsCounter skipCounter(this);
-  nop();
+  Assembler::emit_int32(Assembler::CMPLI_OPCODE | Assembler::opp_u_field(1, 9, 9));
+  assert(is_post_call_nop(*(int*)(pc() - 4)), "post call not not found");
 }
 
 void MacroAssembler::call_VM_base(Register oop_result,

src/hotspot/cpu/ppc/macroAssembler_ppc.hpp

@@ -417,6 +417,12 @@ class MacroAssembler: public Assembler {
   inline void call_stub_and_return_to(Register function_entry, Register return_pc);
 
   void post_call_nop();
+  static bool is_post_call_nop(int instr_bits) {
+    const uint32_t nineth_bit = opp_u_field(1, 9, 9);
+    const uint32_t opcode_mask = 0b111110 << OPCODE_SHIFT;
+    const uint32_t pcn_mask = opcode_mask | nineth_bit;
+    return (instr_bits & pcn_mask) == (Assembler::CMPLI_OPCODE | nineth_bit);
+  }
 
   //
   // Java utilities

src/hotspot/cpu/ppc/nativeInst_ppc.cpp

@@ -429,6 +429,31 @@ void NativePostCallNop::make_deopt() {
   NativeDeoptInstruction::insert(addr_at(0));
 }
 
+bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
+  int32_t i2, i1;
+  assert(is_aligned(cb_offset, 4), "cb offset alignment does not match instruction alignment");
+  assert(!decode(i1, i2), "already patched");
+
+  cb_offset = cb_offset >> 2;
+  if (((oopmap_slot & ppc_oopmap_slot_mask) != oopmap_slot) || ((cb_offset & ppc_cb_offset_mask) != cb_offset)) {
+    return false;  // cannot encode
+  }
+  const uint32_t data = oopmap_slot << ppc_cb_offset_bits | cb_offset;
+  const uint32_t lo_data = data & ppc_data_lo_mask;
+  const uint32_t hi_data = data >> ppc_data_lo_bits;
+  const uint32_t nineth_bit = 1 << (31 - 9);
+  uint32_t instr = Assembler::CMPLI_OPCODE | hi_data << ppc_data_hi_shift | nineth_bit | lo_data;
+  *(uint32_t*)addr_at(0) = instr;
+
+  int32_t oopmap_slot_dec, cb_offset_dec;
+  assert(is_post_call_nop(), "pcn not recognized");
+  assert(decode(oopmap_slot_dec, cb_offset_dec), "encoding failed");
+  assert(oopmap_slot == oopmap_slot_dec, "oopmap slot encoding is wrong");
+  assert((cb_offset << 2) == cb_offset_dec, "cb offset encoding is wrong");
+
+  return true;  // encoding succeeded
+}
+
 void NativeDeoptInstruction::verify() {
 }
 

src/hotspot/cpu/ppc/nativeInst_ppc.hpp

@@ -51,7 +51,7 @@ class NativeInstruction {
   friend class Relocation;
 
  public:
-  bool is_nop() const { return Assembler::is_nop(long_at(0)); }
+  bool is_post_call_nop() const { return MacroAssembler::is_post_call_nop(long_at(0)); }
 
   bool is_jump() const { return Assembler::is_b(long_at(0)); } // See NativeGeneralJump.
 
@@ -506,10 +506,50 @@ class NativeMovRegMem: public NativeInstruction {
 };
 
 class NativePostCallNop: public NativeInstruction {
+
+    // We use CMPI/CMPLI to represent Post Call Nops (PCN)
+
+    //   Bit |0         5|6    |9 |10|11     |16                          31|
+    //       +--------------------------------------------------------------+
+    // Field |OPCODE     |BF   |/ |L |RA     |SI                            |
+    //       +--------------------------------------------------------------+
+    //       |0 0 1 0 1|DATA HI| 1|        DATA LO                          |
+    //       |         |4 bits |  |        22 bits                          |
+    //
+    // Bit 9 is always 1 for PCNs to distinguish them from regular CMPI/CMPLI
+    //
+    // Using both, CMPLI (opcode 10 = 0b001010) and CMPI (opcode 11 = 0b001011) for
+    // PCNs allows using bit 5 from the opcode to encode DATA HI.
+
+    enum {
+      ppc_data_lo_bits = 31 - 9,
+      ppc_data_lo_mask = right_n_bits(ppc_data_lo_bits),
+      ppc_data_hi_bits = 9 - 5,
+      ppc_data_hi_shift = ppc_data_lo_bits + 1,
+      ppc_data_hi_mask = right_n_bits(ppc_data_hi_bits) << ppc_data_hi_shift,
+      ppc_data_bits = ppc_data_lo_bits + ppc_data_hi_bits,
+
+      ppc_oopmap_slot_bits = 9,
+      ppc_oopmap_slot_mask = right_n_bits(ppc_oopmap_slot_bits),
+      ppc_cb_offset_bits = ppc_data_bits - ppc_oopmap_slot_bits,
+      ppc_cb_offset_mask = right_n_bits(ppc_cb_offset_bits),
+};
+
 public:
-  bool check() const { return is_nop(); }
-  bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; }
-  bool patch(int32_t oopmap_slot, int32_t cb_offset) { return false; }
+  bool check() const { return is_post_call_nop(); }
+  bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const {
+    uint32_t instr_bits = long_at(0);
+    uint32_t data_lo = instr_bits & ppc_data_lo_mask;
+    uint32_t data_hi = (instr_bits & ppc_data_hi_mask) >> 1;
+    uint32_t data = data_hi | data_lo;
+    if (data == 0) {
+      return false; // no data found
+    }
+    cb_offset = (data & ppc_cb_offset_mask) << 2;
+    oopmap_slot = data >> ppc_cb_offset_bits;
+    return true; // decoding succeeded
+  }
+  bool patch(int32_t oopmap_slot, int32_t cb_offset);
   void make_deopt();
 };
 

src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp

@@ -37,7 +37,8 @@ frame JavaThread::pd_last_frame() {
   intptr_t* sp = last_Java_sp();
   address pc = _anchor.last_Java_pc();
 
-  return frame(sp, pc);
+  // Likely the frame of a RuntimeStub.
+  return frame(sp, pc, frame::kind::code_blob);
 }
 
 bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, bool isInJava) {
@@ -50,7 +51,7 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
     // pc can be seen as null because not all writers use store pc + release store sp.
     // Simply discard the sample in this very rare case.
     if (pc == nullptr) return false;
-    *fr_addr = frame(sp, pc);
+    *fr_addr = frame(sp, pc, frame::kind::code_blob);
     return true;
   }
 
@@ -66,7 +67,8 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
       return false;
     }
 
-    frame ret_frame((intptr_t*)uc->uc_mcontext.jmp_context.gpr[1/*REG_SP*/], pc);
+    // pc could refer to a native address outside the code cache even though the thread isInJava.
+    frame ret_frame((intptr_t*)uc->uc_mcontext.jmp_context.gpr[1/*REG_SP*/], pc, frame::kind::unknown);
 
     if (ret_frame.fp() == nullptr) {
       // The found frame does not have a valid frame pointer.

src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp

@@ -126,7 +126,7 @@ frame os::fetch_frame_from_context(const void* ucVoid) {
   address epc = fetch_frame_from_context(ucVoid, &sp, &fp);
   // Avoid crash during crash if pc broken.
   if (epc) {
-    frame fr(sp, epc);
+    frame fr(sp, epc, frame::kind::unknown);
     return fr;
   }
   frame fr(sp);
@@ -137,21 +137,21 @@ frame os::fetch_compiled_frame_from_context(const void* ucVoid) {
   const ucontext_t* uc = (const ucontext_t*)ucVoid;
   intptr_t* sp = os::Aix::ucontext_get_sp(uc);
   address lr = ucontext_get_lr(uc);
-  return frame(sp, lr);
+  return frame(sp, lr, frame::kind::unknown);
 }
 
 frame os::get_sender_for_C_frame(frame* fr) {
   if (*fr->sp() == (intptr_t) nullptr) {
     // fr is the last C frame
-    return frame(nullptr, nullptr);
+    return frame();
   }
-  return frame(fr->sender_sp(), fr->sender_pc());
+  return frame(fr->sender_sp(), fr->sender_pc(), frame::kind::unknown);
 }
 
 
 frame os::current_frame() {
   intptr_t* csp = *(intptr_t**) __builtin_frame_address(0);
-  frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame));
+  frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame), frame::kind::unknown);
   return os::get_sender_for_C_frame(&topframe);
 }
 

src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp

@@ -36,7 +36,8 @@ frame JavaThread::pd_last_frame() {
   intptr_t* sp = last_Java_sp();
   address pc = _anchor.last_Java_pc();
 
-  return frame(sp, pc);
+  // Likely the frame of a RuntimeStub.
+  return frame(sp, pc, frame::kind::code_blob);
 }
 
 bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext, bool isInJava) {
@@ -49,7 +50,7 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
     // pc can be seen as null because not all writers use store pc + release store sp.
     // Simply discard the sample in this very rare case.
     if (pc == nullptr) return false;
-    *fr_addr = frame(sp, pc);
+    *fr_addr = frame(sp, pc, frame::kind::code_blob);
     return true;
   }
 
@@ -65,7 +66,8 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
       return false;
     }
 
-    frame ret_frame((intptr_t*)uc->uc_mcontext.regs->gpr[1/*REG_SP*/], pc);
+    // pc could refer to a native address outside the code cache even though the thread isInJava.
+    frame ret_frame((intptr_t*)uc->uc_mcontext.regs->gpr[1/*REG_SP*/], pc, frame::kind::unknown);
 
     if (ret_frame.fp() == nullptr) {
       // The found frame does not have a valid frame pointer.

src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp

@@ -156,28 +156,28 @@ frame os::fetch_frame_from_context(const void* ucVoid) {
   intptr_t* sp;
   intptr_t* fp;
   address epc = fetch_frame_from_context(ucVoid, &sp, &fp);
-  return frame(sp, epc);
+  return frame(sp, epc, frame::kind::unknown);
 }
 
 frame os::fetch_compiled_frame_from_context(const void* ucVoid) {
   const ucontext_t* uc = (const ucontext_t*)ucVoid;
   intptr_t* sp = os::Linux::ucontext_get_sp(uc);
   address lr = ucontext_get_lr(uc);
-  return frame(sp, lr);
+  return frame(sp, lr, frame::kind::unknown);
 }
 
 frame os::get_sender_for_C_frame(frame* fr) {
   if (*fr->sp() == 0) {
     // fr is the last C frame
-    return frame(nullptr, nullptr);
+    return frame();
   }
-  return frame(fr->sender_sp(), fr->sender_pc());
+  return frame(fr->sender_sp(), fr->sender_pc(), frame::kind::unknown);
 }
 
 
 frame os::current_frame() {
   intptr_t* csp = *(intptr_t**) __builtin_frame_address(0);
-  frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame));
+  frame topframe(csp, CAST_FROM_FN_PTR(address, os::current_frame), frame::kind::unknown);
   return os::get_sender_for_C_frame(&topframe);
 }