8337702: Use new ForwardExceptionNode to call StubRoutines::forward_exception_entry()

Reviewed-by: thartmann

src/hotspot/cpu/aarch64/aarch64.ad

@@ -16184,6 +16184,19 @@ instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
   ins_pipe(pipe_class_call);
 %}
 
+// Forward exception.
+instruct ForwardExceptionjmp()
+%{
+  match(ForwardException);
+  ins_cost(CALL_COST);
+
+  format %{ "b forward_exception_stub" %}
+  ins_encode %{
+    __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
+  %}
+  ins_pipe(pipe_class_call);
+%}
+
 // Create exception oop: created by stack-crawling runtime code.
 // Created exception is now available to this handler, and is setup
 // just prior to jumping to this handler. No code emitted.

src/hotspot/cpu/arm/arm.ad

@@ -8953,6 +8953,20 @@ instruct tailjmpInd(IPRegP jump_target, RExceptionRegP ex_oop) %{
   ins_pipe(tail_call);
 %}
 
+// Forward exception.
+instruct ForwardExceptionjmp()
+%{
+  match(ForwardException);
+  ins_cost(CALL_COST);
+
+  format %{ "b    forward_exception_stub" %}
+  ins_encode %{
+    // OK to trash Rtemp, because Rtemp is used by stub
+    __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
+  %}
+  ins_pipe(tail_call);
+%}
+
 // Create exception oop: created by stack-crawling runtime code.
 // Created exception is now available to this handler, and is setup
 // just prior to jumping to this handler.  No code emitted.

src/hotspot/cpu/ppc/ppc.ad

@@ -14534,6 +14534,21 @@ instruct tailjmpInd(iRegPdstNoScratch jump_target, rarg1RegP ex_oop) %{
   ins_pipe(pipe_class_call);
 %}
 
+// Forward exception.
+instruct ForwardExceptionjmp()
+%{
+  match(ForwardException);
+  ins_cost(CALL_COST);
+
+  format %{ "Jmp     forward_exception_stub" %}
+  ins_encode %{
+    __ set_inst_mark();
+    __ b64_patchable(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
+    __ clear_inst_mark();
+  %}
+  ins_pipe(pipe_class_call);
+%}
+
 // Create exception oop: created by stack-crawling runtime code.
 // Created exception is now available to this handler, and is setup
 // just prior to jumping to this handler. No code emitted.

src/hotspot/cpu/riscv/riscv.ad

@@ -10617,6 +10617,22 @@ instruct TailjmpInd(iRegPNoSpNoFp jump_target, iRegP_R10 ex_oop)
   ins_pipe(pipe_class_call);
 %}
 
+// Forward exception.
+instruct ForwardExceptionjmp()
+%{
+  match(ForwardException);
+
+  ins_cost(BRANCH_COST);
+
+  format %{ "j forward_exception_stub\t#@ForwardException" %}
+
+  ins_encode %{
+    __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
+  %}
+
+  ins_pipe(pipe_class_call);
+%}
+
 // Create exception oop: created by stack-crawling runtime code.
 // Created exception is now available to this handler, and is setup
 // just prior to jumping to this handler. No code emitted.

src/hotspot/cpu/s390/s390.ad

@@ -9517,6 +9517,20 @@ instruct tailjmpInd(iRegP jump_target, rarg1RegP ex_oop) %{
   ins_pipe(pipe_class_dummy);
 %}
 
+// Forward exception.
+instruct ForwardExceptionjmp() %{
+  match(ForwardException);
+  ins_cost(CALL_COST);
+  format %{ "Jmp    forward_exception_stub" %}
+  ins_encode %{
+    __ set_inst_mark();
+    __ load_const_optimized(Z_R1_scratch, (address)StubRoutines::forward_exception_entry());
+    __ z_br(Z_R1_scratch);
+    __ clear_inst_mark();
+  %}
+  ins_pipe(pipe_class_dummy);
+%}
+
 // Create exception oop: created by stack-crawling runtime code.
 // Created exception is now available to this handler, and is setup
 // just prior to jumping to this handler. No code emitted.

src/hotspot/cpu/x86/x86_32.ad

@@ -13583,6 +13583,18 @@ instruct tailjmpInd(eRegP_no_EBP jump_target, eAXRegP ex_oop) %{
   ins_pipe( pipe_jmp );
 %}
 
+// Forward exception.
+instruct ForwardExceptionjmp()
+%{
+  match(ForwardException);
+
+  format %{ "JMP    forward_exception_stub" %}
+  ins_encode %{
+    __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
+  %}
+  ins_pipe(pipe_jmp);
+%}
+
 // Create exception oop: created by stack-crawling runtime code.
 // Created exception is now available to this handler, and is setup
 // just prior to jumping to this handler.  No code emitted.

src/hotspot/cpu/x86/x86_64.ad

@@ -12592,6 +12592,18 @@ instruct tailjmpInd(no_rbp_RegP jump_target, rax_RegP ex_oop)
   ins_pipe(pipe_jmp);
 %}
 
+// Forward exception.
+instruct ForwardExceptionjmp()
+%{
+  match(ForwardException);
+
+  format %{ "jmp     forward_exception_stub" %}
+  ins_encode %{
+    __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
+  %}
+  ins_pipe(pipe_jmp);
+%}
+
 // Create exception oop: created by stack-crawling runtime code.
 // Created exception is now available to this handler, and is setup
 // just prior to jumping to this handler.  No code emitted.

src/hotspot/share/adlc/formssel.cpp

@@ -210,15 +210,16 @@ bool InstructForm::is_pinned(FormDict &globals) {
   if ( ! _matrule)  return false;
 
   int  index   = 0;
-  if (_matrule->find_type("Goto",          index)) return true;
+  if (_matrule->find_type("Goto",             index)) return true;
-  if (_matrule->find_type("If",            index)) return true;
+  if (_matrule->find_type("If",               index)) return true;
-  if (_matrule->find_type("CountedLoopEnd",index)) return true;
+  if (_matrule->find_type("CountedLoopEnd",   index)) return true;
-  if (_matrule->find_type("Return",        index)) return true;
+  if (_matrule->find_type("Return",           index)) return true;
-  if (_matrule->find_type("Rethrow",       index)) return true;
+  if (_matrule->find_type("Rethrow",          index)) return true;
-  if (_matrule->find_type("TailCall",      index)) return true;
+  if (_matrule->find_type("TailCall",         index)) return true;
-  if (_matrule->find_type("TailJump",      index)) return true;
+  if (_matrule->find_type("TailJump",         index)) return true;
-  if (_matrule->find_type("Halt",          index)) return true;
+  if (_matrule->find_type("ForwardException", index)) return true;
-  if (_matrule->find_type("Jump",          index)) return true;
+  if (_matrule->find_type("Halt",             index)) return true;
+  if (_matrule->find_type("Jump",             index)) return true;
 
   return is_parm(globals);
 }
@@ -228,12 +229,13 @@ bool InstructForm::is_projection(FormDict &globals) {
   if ( ! _matrule)  return false;
 
   int  index   = 0;
-  if (_matrule->find_type("Goto",    index)) return true;
+  if (_matrule->find_type("Goto",             index)) return true;
-  if (_matrule->find_type("Return",  index)) return true;
+  if (_matrule->find_type("Return",           index)) return true;
-  if (_matrule->find_type("Rethrow", index)) return true;
+  if (_matrule->find_type("Rethrow",          index)) return true;
-  if (_matrule->find_type("TailCall",index)) return true;
+  if (_matrule->find_type("TailCall",         index)) return true;
-  if (_matrule->find_type("TailJump",index)) return true;
+  if (_matrule->find_type("TailJump",         index)) return true;
-  if (_matrule->find_type("Halt",    index)) return true;
+  if (_matrule->find_type("ForwardException", index)) return true;
+  if (_matrule->find_type("Halt",             index)) return true;
 
   return false;
 }
@@ -376,6 +378,7 @@ bool InstructForm::is_ideal_return() const {
   if (_matrule->find_type("Rethrow",index)) return true;
   if (_matrule->find_type("TailCall",index)) return true;
   if (_matrule->find_type("TailJump",index)) return true;
+  if (_matrule->find_type("ForwardException", index)) return true;
 
   return false;
 }
@@ -894,6 +897,7 @@ uint InstructForm::oper_input_base(FormDict &globals) {
       strcmp(_matrule->_opType,"Rethrow"   )==0 ||
       strcmp(_matrule->_opType,"TailCall"  )==0 ||
       strcmp(_matrule->_opType,"TailJump"  )==0 ||
+      strcmp(_matrule->_opType,"ForwardException")==0 ||
       strcmp(_matrule->_opType,"SafePoint" )==0 ||
       strcmp(_matrule->_opType,"Halt"      )==0 )
     return AdlcVMDeps::Parms;   // Skip the machine-state edges

src/hotspot/share/opto/callnode.hpp

@@ -152,6 +152,17 @@ class RethrowNode : public Node {
 };
 
 
+//------------------------------ForwardExceptionNode---------------------------
+// Pop stack frame and jump to StubRoutines::forward_exception_entry()
+class ForwardExceptionNode : public ReturnNode {
+public:
+  ForwardExceptionNode(Node* cntrl, Node* i_o, Node* memory, Node* frameptr, Node* retadr)
+    : ReturnNode(TypeFunc::Parms, cntrl, i_o, memory, frameptr, retadr) {
+  }
+
+  virtual int Opcode() const;
+};
+
 //------------------------------TailCallNode-----------------------------------
 // Pop stack frame and jump indirect
 class TailCallNode : public ReturnNode {

src/hotspot/share/opto/classes.hpp

@@ -184,6 +184,7 @@ macro(FastLock)
 macro(FastUnlock)
 macro(FmaD)
 macro(FmaF)
+macro(ForwardException)
 macro(Goto)
 macro(Halt)
 macro(CountPositives)

src/hotspot/share/opto/domgraph.cpp

@@ -239,6 +239,7 @@ uint Block_Stack::most_frequent_successor( Block *b ) {
     break;
   case Op_TailCall:
   case Op_TailJump:
+  case Op_ForwardException:
   case Op_Return:
   case Op_Halt:
   case Op_Rethrow:

src/hotspot/share/opto/gcm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2031,6 +2031,7 @@ float Block::succ_prob(uint i) {
 
   case Op_TailCall:
   case Op_TailJump:
+  case Op_ForwardException:
   case Op_Return:
   case Op_Halt:
   case Op_Rethrow:
@@ -2084,6 +2085,7 @@ int Block::num_fall_throughs() {
   case Op_NeverBranch:
   case Op_TailCall:
   case Op_TailJump:
+  case Op_ForwardException:
   case Op_Return:
   case Op_Halt:
   case Op_Rethrow:
@@ -2129,6 +2131,7 @@ bool Block::succ_fall_through(uint i) {
   case Op_NeverBranch:
   case Op_TailCall:
   case Op_TailJump:
+  case Op_ForwardException:
   case Op_Return:
   case Op_Halt:
   case Op_Rethrow:

src/hotspot/share/opto/generateOptoStub.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -255,13 +255,11 @@ void GraphKit::gen_stub(address C_function,
   Node* if_not_null = _gvn.transform( new IfTrueNode(iff)  );
 
   assert (StubRoutines::forward_exception_entry() != nullptr, "must be generated before");
-  Node *exc_target = makecon(TypeRawPtr::make( StubRoutines::forward_exception_entry() ));
+  Node *to_exc = new ForwardExceptionNode(if_not_null,
-  Node *to_exc = new TailCallNode(if_not_null,
+                                          i_o(),
-                                  i_o(),
+                                          exit_memory,
-                                  exit_memory,
+                                          frameptr(),
-                                  frameptr(),
+                                          returnadr());
-                                  returnadr(),
-                                  exc_target, null());
   root()->add_req(_gvn.transform(to_exc));  // bind to root to keep live
   C->init_start(start);
 

src/hotspot/share/opto/matcher.cpp

@@ -804,6 +804,10 @@ void Matcher::Fixup_Save_On_Entry( ) {
   if (ret_edge_cnt > TypeFunc::Parms)
     ret_rms[TypeFunc::Parms+0] = _return_value_mask;
 
+  // Input RegMask array shared by all ForwardExceptions
+  uint forw_exc_edge_cnt = TypeFunc::Parms;
+  RegMask* forw_exc_rms  = init_input_masks( forw_exc_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
+
   // Input RegMask array shared by all Rethrows.
   uint reth_edge_cnt = TypeFunc::Parms+1;
   RegMask *reth_rms  = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
@@ -863,6 +867,7 @@ void Matcher::Fixup_Save_On_Entry( ) {
       case Op_Rethrow  : exit->_in_rms = reth_rms; break;
       case Op_TailCall : exit->_in_rms = tail_call_rms; break;
       case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
+      case Op_ForwardException: exit->_in_rms = forw_exc_rms; break;
       case Op_Halt     : exit->_in_rms = halt_rms; break;
       default          : ShouldNotReachHere();
     }
@@ -882,6 +887,7 @@ void Matcher::Fixup_Save_On_Entry( ) {
       reth_rms     [     reth_edge_cnt] = mreg2regmask[i];
       tail_call_rms[tail_call_edge_cnt] = mreg2regmask[i];
       tail_jump_rms[tail_jump_edge_cnt] = mreg2regmask[i];
+      forw_exc_rms [ forw_exc_edge_cnt] = mreg2regmask[i];
       // Halts need the SOE registers, but only in the stack as debug info.
       // A just-prior uncommon-trap or deoptimization will use the SOE regs.
       halt_rms     [     halt_edge_cnt] = *idealreg2spillmask[_register_save_type[i]];
@@ -899,6 +905,7 @@ void Matcher::Fixup_Save_On_Entry( ) {
         reth_rms     [     reth_edge_cnt].Insert(OptoReg::Name(i+1));
         tail_call_rms[tail_call_edge_cnt].Insert(OptoReg::Name(i+1));
         tail_jump_rms[tail_jump_edge_cnt].Insert(OptoReg::Name(i+1));
+        forw_exc_rms [ forw_exc_edge_cnt].Insert(OptoReg::Name(i+1));
         halt_rms     [     halt_edge_cnt].Insert(OptoReg::Name(i+1));
         mproj = new MachProjNode( start, proj_cnt, ret_rms[ret_edge_cnt], Op_RegD );
         proj_cnt += 2;          // Skip 2 for doubles
@@ -911,6 +918,7 @@ void Matcher::Fixup_Save_On_Entry( ) {
         reth_rms     [     reth_edge_cnt] = RegMask::Empty;
         tail_call_rms[tail_call_edge_cnt] = RegMask::Empty;
         tail_jump_rms[tail_jump_edge_cnt] = RegMask::Empty;
+        forw_exc_rms [ forw_exc_edge_cnt] = RegMask::Empty;
         halt_rms     [     halt_edge_cnt] = RegMask::Empty;
         mproj = C->top();
       }
@@ -925,6 +933,7 @@ void Matcher::Fixup_Save_On_Entry( ) {
         reth_rms     [     reth_edge_cnt].Insert(OptoReg::Name(i+1));
         tail_call_rms[tail_call_edge_cnt].Insert(OptoReg::Name(i+1));
         tail_jump_rms[tail_jump_edge_cnt].Insert(OptoReg::Name(i+1));
+        forw_exc_rms [ forw_exc_edge_cnt].Insert(OptoReg::Name(i+1));
         halt_rms     [     halt_edge_cnt].Insert(OptoReg::Name(i+1));
         mproj = new MachProjNode( start, proj_cnt, ret_rms[ret_edge_cnt], Op_RegL );
         proj_cnt += 2;          // Skip 2 for longs
@@ -937,6 +946,7 @@ void Matcher::Fixup_Save_On_Entry( ) {
         reth_rms     [     reth_edge_cnt] = RegMask::Empty;
         tail_call_rms[tail_call_edge_cnt] = RegMask::Empty;
         tail_jump_rms[tail_jump_edge_cnt] = RegMask::Empty;
+        forw_exc_rms [ forw_exc_edge_cnt] = RegMask::Empty;
         halt_rms     [     halt_edge_cnt] = RegMask::Empty;
         mproj = C->top();
       } else {
@@ -948,11 +958,13 @@ void Matcher::Fixup_Save_On_Entry( ) {
       reth_edge_cnt ++;
       tail_call_edge_cnt ++;
       tail_jump_edge_cnt ++;
+      forw_exc_edge_cnt++;
       halt_edge_cnt ++;
 
       // Add a use of the SOE register to all exit paths
-      for( uint j=1; j < root->req(); j++ )
+      for (uint j=1; j < root->req(); j++) {
         root->in(j)->add_req(mproj);
+      }
     } // End of if a save-on-entry register
   } // End of for all machine registers
 }
@@ -1070,6 +1082,7 @@ static void match_alias_type(Compile* C, Node* n, Node* m) {
     case Op_Halt:
     case Op_TailCall:
     case Op_TailJump:
+    case Op_ForwardException:
       nidx = Compile::AliasIdxBot;
       nat = TypePtr::BOTTOM;
       break;

src/hotspot/share/runtime/vmStructs.cpp

@@ -1422,6 +1422,7 @@
   declare_c2_type(RethrowNode, Node)                                      \
   declare_c2_type(TailCallNode, ReturnNode)                               \
   declare_c2_type(TailJumpNode, ReturnNode)                               \
+  declare_c2_type(ForwardExceptionNode, ReturnNode)                       \
   declare_c2_type(SafePointNode, MultiNode)                               \
   declare_c2_type(CallNode, SafePointNode)                                \
   declare_c2_type(CallJavaNode, CallNode)                                 \