8254693: Add Panama feature to pass heap segments to native code

Reviewed-by: mcimadamore, lucy, vlivanov

src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp

@@ -36,63 +36,6 @@
 
 #define __ _masm->
 
-class DowncallStubGenerator : public StubCodeGenerator {
-  BasicType* _signature;
-  int _num_args;
-  BasicType _ret_bt;
-  const ABIDescriptor& _abi;
-
-  const GrowableArray<VMStorage>& _input_registers;
-  const GrowableArray<VMStorage>& _output_registers;
-
-  bool _needs_return_buffer;
-  int _captured_state_mask;
-  bool _needs_transition;
-
-  int _frame_complete;
-  int _frame_size_slots;
-  OopMapSet* _oop_maps;
-public:
-  DowncallStubGenerator(CodeBuffer* buffer,
-                         BasicType* signature,
-                         int num_args,
-                         BasicType ret_bt,
-                         const ABIDescriptor& abi,
-                         const GrowableArray<VMStorage>& input_registers,
-                         const GrowableArray<VMStorage>& output_registers,
-                         bool needs_return_buffer,
-                         int captured_state_mask,
-                         bool needs_transition)
-   : StubCodeGenerator(buffer, PrintMethodHandleStubs),
-     _signature(signature),
-     _num_args(num_args),
-     _ret_bt(ret_bt),
-     _abi(abi),
-     _input_registers(input_registers),
-     _output_registers(output_registers),
-     _needs_return_buffer(needs_return_buffer),
-     _captured_state_mask(captured_state_mask),
-     _needs_transition(needs_transition),
-     _frame_complete(0),
-     _frame_size_slots(0),
-     _oop_maps(nullptr) {
-  }
-
-  void generate();
-
-  int frame_complete() const {
-    return _frame_complete;
-  }
-
-  int framesize() const {
-    return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt));
-  }
-
-  OopMapSet* oop_maps() const {
-    return _oop_maps;
-  }
-};
-
 static const int native_invoker_code_base_size = 256;
 static const int native_invoker_size_per_arg = 8;
 
@@ -108,10 +51,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   int code_size = native_invoker_code_base_size + (num_args * native_invoker_size_per_arg);
   int locs_size = 1; // must be non-zero
   CodeBuffer code("nep_invoker_blob", code_size, locs_size);
-  DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi,
+  StubGenerator g(&code, signature, num_args, ret_bt, abi,
-                          input_registers, output_registers,
+                  input_registers, output_registers,
-                          needs_return_buffer, captured_state_mask,
+                  needs_return_buffer, captured_state_mask,
-                          needs_transition);
+                  needs_transition);
   g.generate();
   code.log_section_sizes("nep_invoker_blob");
 
@@ -134,7 +77,39 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   return stub;
 }
 
-void DowncallStubGenerator::generate() {
+static constexpr int RFP_BIAS = 16; // skip old rbp and return address
+
+void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, VMStorage tmp1, VMStorage tmp2) const {
+  Register r_tmp1 = as_Register(tmp1);
+  Register r_tmp2 = as_Register(tmp2);
+  if (reg_oop.is_reg()) {
+    assert(reg_oop.type() == StorageType::INTEGER, "expected");
+    Register reg_oop_reg = as_Register(reg_oop);
+    if (reg_offset.is_reg()) {
+      assert(reg_offset.type() == StorageType::INTEGER, "expected");
+      __ add(reg_oop_reg, reg_oop_reg, as_Register(reg_offset));
+    } else {
+      assert(reg_offset.is_stack(), "expected");
+      assert(reg_offset.stack_size() == 8, "expected long");
+      Address offset_addr(rfp, RFP_BIAS + reg_offset.offset());
+      __ ldr (r_tmp1, offset_addr);
+      __ add(reg_oop_reg, reg_oop_reg, r_tmp1);
+    }
+  } else {
+    assert(reg_oop.is_stack(), "expected");
+    assert(reg_oop.stack_size() == 8, "expected long");
+    assert(reg_offset.is_stack(), "expected");
+    assert(reg_offset.stack_size() == 8, "expected long");
+    Address offset_addr(rfp, RFP_BIAS + reg_offset.offset());
+    Address oop_addr(rfp, RFP_BIAS + reg_oop.offset());
+    __ ldr(r_tmp1, offset_addr);
+    __ ldr(r_tmp2, oop_addr);
+    __ add(r_tmp1, r_tmp1, r_tmp2);
+    __ str(r_tmp1, oop_addr);
+  }
+}
+
+void DowncallLinker::StubGenerator::generate() {
   enum layout {
     rfp_off,
     rfp_off2,
@@ -150,23 +125,16 @@ void DowncallStubGenerator::generate() {
   Register tmp1 = r9;
   Register tmp2 = r10;
 
-  VMStorage shuffle_reg = as_VMStorage(r19);
+  GrowableArray<VMStorage> java_regs;
-  JavaCallingConvention in_conv;
+  ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs);
-  NativeCallingConvention out_conv(_input_registers);
+  bool has_objects = false;
-  ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg);
+  GrowableArray<VMStorage> filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature,
-
+                                                                                             _num_args, has_objects);
-#ifndef PRODUCT
+  assert(!(_needs_transition && has_objects), "can not pass objects when doing transition");
-  LogTarget(Trace, foreign, downcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
 
   int allocated_frame_size = 0;
   assert(_abi._shadow_space_bytes == 0, "not expecting shadow space on AArch64");
-  allocated_frame_size += arg_shuffle.out_arg_bytes();
+  allocated_frame_size += ForeignGlobals::compute_out_arg_bytes(_input_registers);
 
   bool should_save_return_value = !_needs_return_buffer;
   RegSpiller out_reg_spiller(_output_registers);
@@ -193,6 +161,33 @@ void DowncallStubGenerator::generate() {
     allocated_frame_size += BytesPerWord;
   }
 
+  // The space we have allocated will look like:
+  //
+  // FP-> |                     |
+  //      |---------------------| = frame_bottom_offset = frame_size
+  //      | (optional)          |
+  //      | capture state buf   |
+  //      |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
+  //      | (optional)          |
+  //      | return buffer       |
+  //      |---------------------| = StubLocations::RETURN_BUFFER
+  // SP-> | out/stack args      | or | out_reg_spiller area |
+  //
+  // Note how the last chunk can be shared, since the 3 uses occur at different times.
+
+  VMStorage shuffle_reg = as_VMStorage(r19);
+  GrowableArray<VMStorage> out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
+  ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, downcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   _frame_size_slots = align_up(framesize + (allocated_frame_size >> LogBytesPerInt), 4);
   assert(is_even(_frame_size_slots/2), "sp not 16-byte aligned");
 
@@ -218,8 +213,12 @@ void DowncallStubGenerator::generate() {
     __ stlrw(tmp1, tmp2);
   }
 
+  if (has_objects) {
+    add_offsets_to_oops(java_regs, as_VMStorage(tmp1), as_VMStorage(tmp2));
+  }
+
   __ block_comment("{ argument shuffle");
-  arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes, locs);
+  arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
   __ block_comment("} argument shuffle");
 
   __ blr(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));

src/hotspot/cpu/aarch64/foreignGlobals_aarch64.cpp

@@ -200,21 +200,13 @@ static void move_v128(MacroAssembler* masm, int out_stk_bias,
   }
 }
 
-void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
   Register tmp_reg = as_Register(tmp);
   for (int i = 0; i < _moves.length(); i++) {
     Move move = _moves.at(i);
     VMStorage from_reg = move.from;
     VMStorage to_reg   = move.to;
 
-    // replace any placeholders
-    if (from_reg.type() == StorageType::PLACEHOLDER) {
-      from_reg = locs.get(from_reg);
-    }
-    if (to_reg.type() == StorageType::PLACEHOLDER) {
-      to_reg = locs.get(to_reg);
-    }
-
     switch (from_reg.type()) {
       case StorageType::INTEGER:
         assert(from_reg.segment_mask() == REG64_MASK, "only 64-bit register supported");

src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp

@@ -118,7 +118,6 @@ static const int upcall_stub_code_base_size = 1024;
 static const int upcall_stub_size_per_arg = 16;
 
 address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
-                                       BasicType* in_sig_bt, int total_in_args,
                                        BasicType* out_sig_bt, int total_out_args,
                                        BasicType ret_type,
                                        jobject jabi, jobject jconv,
@@ -126,26 +125,15 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   ResourceMark rm;
   const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi);
   const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv);
-  int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg);
+  int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg);
   CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1);
 
-  Register shuffle_reg = r19;
+  GrowableArray<VMStorage> unfiltered_out_regs;
-  JavaCallingConvention out_conv;
+  int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs);
-  NativeCallingConvention in_conv(call_regs._arg_regs);
-  ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, as_VMStorage(shuffle_reg));
   int preserved_bytes = SharedRuntime::out_preserve_stack_slots() * VMRegImpl::stack_slot_size;
-  int stack_bytes = preserved_bytes + arg_shuffle.out_arg_bytes();
+  int stack_bytes = preserved_bytes + out_arg_bytes;
   int out_arg_area = align_up(stack_bytes , StackAlignmentInBytes);
 
-#ifndef PRODUCT
-  LogTarget(Trace, foreign, upcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
-
   // out_arg_area (for stack arguments) doubles as shadow space for native calls.
   // make sure it is big enough.
   if (out_arg_area < frame::arg_reg_save_area_bytes) {
@@ -153,13 +141,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   }
 
   int reg_save_area_size = compute_reg_save_area_size(abi);
-  RegSpiller arg_spilller(call_regs._arg_regs);
+  RegSpiller arg_spiller(call_regs._arg_regs);
   RegSpiller result_spiller(call_regs._ret_regs);
 
   int shuffle_area_offset    = 0;
   int res_save_area_offset   = shuffle_area_offset    + out_arg_area;
   int arg_save_area_offset   = res_save_area_offset   + result_spiller.spill_size_bytes();
-  int reg_save_area_offset   = arg_save_area_offset   + arg_spilller.spill_size_bytes();
+  int reg_save_area_offset   = arg_save_area_offset   + arg_spiller.spill_size_bytes();
   int frame_data_offset      = reg_save_area_offset   + reg_save_area_size;
   int frame_bottom_offset    = frame_data_offset      + sizeof(UpcallStub::FrameData);
 
@@ -173,6 +161,20 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
     locs.set(StubLocations::RETURN_BUFFER, abi._scratch1);
   }
 
+  Register shuffle_reg = r19;
+  GrowableArray<VMStorage> in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs);
+  GrowableArray<VMStorage> filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs);
+  ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, as_VMStorage(shuffle_reg));
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, upcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   int frame_size = frame_bottom_offset;
   frame_size = align_up(frame_size, StackAlignmentInBytes);
 
@@ -212,7 +214,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
 
   // we have to always spill args since we need to do a call to get the thread
   // (and maybe attach it).
-  arg_spilller.generate_spill(_masm, arg_save_area_offset);
+  arg_spiller.generate_spill(_masm, arg_save_area_offset);
   preserve_callee_saved_registers(_masm, abi, reg_save_area_offset);
 
   __ block_comment("{ on_entry");
@@ -225,12 +227,12 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   __ block_comment("} on_entry");
 
   __ block_comment("{ argument shuffle");
-  arg_spilller.generate_fill(_masm, arg_save_area_offset);
+  arg_spiller.generate_fill(_masm, arg_save_area_offset);
   if (needs_return_buffer) {
     assert(ret_buf_offset != -1, "no return buffer allocated");
     __ lea(as_Register(locs.get(StubLocations::RETURN_BUFFER)), Address(sp, ret_buf_offset));
   }
-  arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0, locs);
+  arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0);
   __ block_comment("} argument shuffle");
 
   __ block_comment("{ receiver ");

src/hotspot/cpu/arm/downcallLinker_arm.cpp

@@ -38,3 +38,8 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   Unimplemented();
   return nullptr;
 }
+
+void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset,
+                                                         VMStorage tmp1, VMStorage tmp2) const {
+  Unimplemented();
+}

src/hotspot/cpu/arm/foreignGlobals_arm.cpp

@@ -51,6 +51,6 @@ void RegSpiller::pd_load_reg(MacroAssembler* masm, int offset, VMStorage reg) {
   Unimplemented();
 }
 
-void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
   Unimplemented();
 }

src/hotspot/cpu/arm/upcallLinker_arm.cpp

@@ -26,7 +26,6 @@
 #include "utilities/debug.hpp"
 
 address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
-                                       BasicType* in_sig_bt, int total_in_args,
                                        BasicType* out_sig_bt, int total_out_args,
                                        BasicType ret_type,
                                        jobject jabi, jobject jconv,

src/hotspot/cpu/ppc/downcallLinker_ppc.cpp

@@ -36,63 +36,6 @@
 
 #define __ _masm->
 
-class DowncallStubGenerator : public StubCodeGenerator {
-  BasicType* _signature;
-  int _num_args;
-  BasicType _ret_bt;
-  const ABIDescriptor& _abi;
-
-  const GrowableArray<VMStorage>& _input_registers;
-  const GrowableArray<VMStorage>& _output_registers;
-
-  bool _needs_return_buffer;
-  int _captured_state_mask;
-  bool _needs_transition;
-
-  int _frame_complete;
-  int _frame_size_slots;
-  OopMapSet* _oop_maps;
-public:
-  DowncallStubGenerator(CodeBuffer* buffer,
-                         BasicType* signature,
-                         int num_args,
-                         BasicType ret_bt,
-                         const ABIDescriptor& abi,
-                         const GrowableArray<VMStorage>& input_registers,
-                         const GrowableArray<VMStorage>& output_registers,
-                         bool needs_return_buffer,
-                         int captured_state_mask,
-                         bool needs_transition)
-   : StubCodeGenerator(buffer, PrintMethodHandleStubs),
-     _signature(signature),
-     _num_args(num_args),
-     _ret_bt(ret_bt),
-     _abi(abi),
-     _input_registers(input_registers),
-     _output_registers(output_registers),
-     _needs_return_buffer(needs_return_buffer),
-     _captured_state_mask(captured_state_mask),
-     _needs_transition(needs_transition),
-     _frame_complete(0),
-     _frame_size_slots(0),
-     _oop_maps(nullptr) {
-  }
-
-  void generate();
-
-  int frame_complete() const {
-    return _frame_complete;
-  }
-
-  int framesize() const {
-    return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt));
-  }
-
-  OopMapSet* oop_maps() const {
-    return _oop_maps;
-  }
-};
-
 static const int native_invoker_code_base_size = 384;
 static const int native_invoker_size_per_arg = 8;
 
@@ -108,10 +51,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   int code_size = native_invoker_code_base_size + (num_args * native_invoker_size_per_arg);
   int locs_size = 1; // must be non-zero
   CodeBuffer code("nep_invoker_blob", code_size, locs_size);
-  DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi,
+  StubGenerator g(&code, signature, num_args, ret_bt, abi,
-                          input_registers, output_registers,
+                  input_registers, output_registers,
-                          needs_return_buffer, captured_state_mask,
+                  needs_return_buffer, captured_state_mask,
-                          needs_transition);
+                  needs_transition);
   g.generate();
   code.log_section_sizes("nep_invoker_blob");
 
@@ -134,23 +77,40 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   return stub;
 }
 
-void DowncallStubGenerator::generate() {
+static constexpr int FP_BIAS = frame::jit_out_preserve_size;
-  Register callerSP            = R2, // C/C++ uses R2 as TOC, but we can reuse it here
+static const Register callerSP = R2; // C/C++ uses R2 as TOC, but we can reuse it here
-           tmp                 = R11_scratch1, // same as shuffle_reg
-           call_target_address = R12_scratch2; // same as _abi._scratch2 (ABIv2 requires this reg!)
-  VMStorage shuffle_reg = _abi._scratch1;
-  JavaCallingConvention in_conv;
-  NativeCallingConvention out_conv(_input_registers);
-  ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg);
 
-#ifndef PRODUCT
+void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset,
-  LogTarget(Trace, foreign, downcall) lt;
+                                                         VMStorage tmp1, VMStorage tmp2) const {
-  if (lt.is_enabled()) {
+  Register r_tmp1 = as_Register(tmp1);
-    ResourceMark rm;
+  Register r_tmp2 = as_Register(tmp2);
-    LogStream ls(lt);
+  if (reg_oop.is_reg()) {
-    arg_shuffle.print_on(&ls);
+    assert(reg_oop.type() == StorageType::INTEGER, "expected");
+    Register reg_oop_reg = as_Register(reg_oop);
+    if (reg_offset.is_reg()) {
+      assert(reg_offset.type() == StorageType::INTEGER, "expected");
+      __ add(reg_oop_reg, reg_oop_reg, as_Register(reg_offset));
+    } else {
+      assert(reg_offset.is_stack(), "expected");
+      assert(reg_offset.stack_size() == 8, "expected long");
+      __ ld(r_tmp1, reg_offset.offset() + FP_BIAS, callerSP);
+      __ add(reg_oop_reg, reg_oop_reg, r_tmp1);
+    }
+  } else {
+    assert(reg_oop.is_stack(), "expected");
+    assert(reg_oop.stack_size() == 8, "expected long");
+    assert(reg_offset.is_stack(), "expected");
+    assert(reg_offset.stack_size() == 8, "expected long");
+    __ ld(r_tmp1, reg_offset.offset() + FP_BIAS, callerSP);
+    __ ld(r_tmp2, reg_oop.offset() + FP_BIAS, callerSP);
+    __ add(r_tmp1, r_tmp2, r_tmp1);
+    __ std(r_tmp1, reg_oop.offset() + FP_BIAS, callerSP);
   }
-#endif
+}
+
+void DowncallLinker::StubGenerator::generate() {
+  Register tmp                 = R11_scratch1, // same as shuffle_reg
+           call_target_address = R12_scratch2; // same as _abi._scratch2 (ABIv2 requires this reg!)
 
   // Stack frame size computation:
   // We use the number of input VMStorage elements because PPC64 requires slots for all arguments
@@ -191,6 +151,26 @@ void DowncallStubGenerator::generate() {
     allocated_frame_size += BytesPerWord;
   }
 
+  GrowableArray<VMStorage> java_regs;
+  ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs);
+  bool has_objects = false;
+  GrowableArray<VMStorage> filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature,
+                                                                                             _num_args, has_objects);
+  assert(!(_needs_transition && has_objects), "can not pass objects when doing transition");
+
+  GrowableArray<VMStorage> out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
+
+  ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, _abi._scratch1);
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, downcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   allocated_frame_size = align_up(allocated_frame_size, StackAlignmentInBytes);
   _frame_size_slots = allocated_frame_size >> LogBytesPerInt;
 
@@ -216,8 +196,12 @@ void DowncallStubGenerator::generate() {
     __ stw(R0, in_bytes(JavaThread::thread_state_offset()), R16_thread);
   }
 
+  if (has_objects) {
+    add_offsets_to_oops(java_regs, _abi._scratch1, _abi._scratch2);
+  }
+
   __ block_comment("{ argument shuffle");
-  arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::jit_out_preserve_size, frame::native_abi_minframe_size, locs);
+  arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::jit_out_preserve_size, frame::native_abi_minframe_size);
   __ block_comment("} argument shuffle");
 
   __ call_c(call_target_address);

src/hotspot/cpu/ppc/foreignGlobals_ppc.cpp

@@ -226,21 +226,13 @@ static void move_stack(MacroAssembler* masm, Register callerSP, int in_stk_bias,
   }
 }
 
-void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
   Register callerSP = as_Register(tmp); // preset
   for (int i = 0; i < _moves.length(); i++) {
     Move move = _moves.at(i);
     VMStorage from_reg = move.from;
     VMStorage to_reg   = move.to;
 
-    // replace any placeholders
-    if (from_reg.type() == StorageType::PLACEHOLDER) {
-      from_reg = locs.get(from_reg);
-    }
-    if (to_reg.type() == StorageType::PLACEHOLDER) {
-      to_reg = locs.get(to_reg);
-    }
-
     switch (from_reg.type()) {
       case StorageType::INTEGER:
         move_reg64(masm, out_stk_bias, from_reg, to_reg);

src/hotspot/cpu/ppc/upcallLinker_ppc.cpp

@@ -119,7 +119,6 @@ static const int upcall_stub_code_base_size = 1024;
 static const int upcall_stub_size_per_arg = 16; // arg save & restore + move
 
 address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
-                                       BasicType* in_sig_bt, int total_in_args,
                                        BasicType* out_sig_bt, int total_out_args,
                                        BasicType ret_type,
                                        jobject jabi, jobject jconv,
@@ -127,27 +126,16 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   ResourceMark rm;
   const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi);
   const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv);
-  int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg);
+  int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg);
   CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1);
 
   Register callerSP            = R2, // C/C++ uses R2 as TOC, but we can reuse it here
            tmp                 = R11_scratch1, // same as shuffle_reg
            call_target_address = R12_scratch2; // same as _abi._scratch2
-  VMStorage shuffle_reg = abi._scratch1;
+  GrowableArray<VMStorage> unfiltered_out_regs;
-  JavaCallingConvention out_conv;
+  int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs);
-  NativeCallingConvention in_conv(call_regs._arg_regs);
-  ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, shuffle_reg);
   // The Java call uses the JIT ABI, but we also call C.
-  int out_arg_area = MAX2(frame::jit_out_preserve_size + arg_shuffle.out_arg_bytes(), (int)frame::native_abi_reg_args_size);
+  int out_arg_area = MAX2(frame::jit_out_preserve_size + out_arg_bytes, (int)frame::native_abi_reg_args_size);
-
-#ifndef PRODUCT
-  LogTarget(Trace, foreign, upcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
 
   int reg_save_area_size = compute_reg_save_area_size(abi);
   RegSpiller arg_spiller(call_regs._arg_regs);
@@ -169,6 +157,19 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
     locs.set(StubLocations::RETURN_BUFFER, abi._scratch2);
   }
 
+  GrowableArray<VMStorage> in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs);
+  GrowableArray<VMStorage> filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs);
+  ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, abi._scratch1);
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, upcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   int frame_size = align_up(frame_bottom_offset, StackAlignmentInBytes);
 
   // The space we have allocated will look like:
@@ -229,7 +230,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
     __ addi(as_Register(locs.get(StubLocations::RETURN_BUFFER)), R1_SP, ret_buf_offset);
   }
   __ ld(callerSP, _abi0(callers_sp), R1_SP); // preset (used to access caller frame argument slots)
-  arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::native_abi_minframe_size, frame::jit_out_preserve_size, locs);
+  arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::native_abi_minframe_size, frame::jit_out_preserve_size);
   __ block_comment("} argument shuffle");
 
   __ block_comment("{ receiver ");

src/hotspot/cpu/riscv/downcallLinker_riscv.cpp

@@ -37,63 +37,6 @@
 
 #define __ _masm->
 
-class DowncallStubGenerator : public StubCodeGenerator {
-  BasicType* _signature;
-  int _num_args;
-  BasicType _ret_bt;
-
-  const ABIDescriptor& _abi;
-  const GrowableArray<VMStorage>& _input_registers;
-  const GrowableArray<VMStorage>& _output_registers;
-
-  bool _needs_return_buffer;
-  int _captured_state_mask;
-  bool _needs_transition;
-
-  int _frame_complete;
-  int _frame_size_slots;
-  OopMapSet* _oop_maps;
-public:
-  DowncallStubGenerator(CodeBuffer* buffer,
-                        BasicType* signature,
-                        int num_args,
-                        BasicType ret_bt,
-                        const ABIDescriptor& abi,
-                        const GrowableArray<VMStorage>& input_registers,
-                        const GrowableArray<VMStorage>& output_registers,
-                        bool needs_return_buffer,
-                        int captured_state_mask,
-                        bool needs_transition)
-   : StubCodeGenerator(buffer, PrintMethodHandleStubs),
-     _signature(signature),
-     _num_args(num_args),
-     _ret_bt(ret_bt),
-     _abi(abi),
-     _input_registers(input_registers),
-     _output_registers(output_registers),
-     _needs_return_buffer(needs_return_buffer),
-     _captured_state_mask(captured_state_mask),
-     _needs_transition(needs_transition),
-     _frame_complete(0),
-     _frame_size_slots(0),
-     _oop_maps(nullptr) {
-  }
-
-  void generate();
-
-  int frame_complete() const {
-    return _frame_complete;
-  }
-
-  int framesize() const {
-    return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt));
-  }
-
-  OopMapSet* oop_maps() const {
-    return _oop_maps;
-  }
-};
-
 static const int native_invoker_code_base_size = 256;
 static const int native_invoker_size_per_arg = 8;
 
@@ -109,10 +52,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   int code_size = native_invoker_code_base_size + (num_args * native_invoker_size_per_arg);
   int locs_size = 1; // must be non-zero
   CodeBuffer code("nep_invoker_blob", code_size, locs_size);
-  DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi,
+  StubGenerator g(&code, signature, num_args, ret_bt, abi,
-                          input_registers, output_registers,
+                  input_registers, output_registers,
-                          needs_return_buffer, captured_state_mask,
+                  needs_return_buffer, captured_state_mask,
-                          needs_transition);
+                  needs_transition);
   g.generate();
   code.log_section_sizes("nep_invoker_blob");
 
@@ -135,7 +78,36 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   return stub;
 }
 
-void DowncallStubGenerator::generate() {
+static constexpr int FP_BIAS = 0; // sender_sp_offset is 0 on RISCV
+
+void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, VMStorage tmp1, VMStorage tmp2) const {
+  Register r_tmp1 = as_Register(tmp1);
+  Register r_tmp2 = as_Register(tmp2);
+  if (reg_oop.is_reg()) {
+    assert(reg_oop.type() == StorageType::INTEGER, "expected");
+    Register reg_oop_reg = as_Register(reg_oop);
+    if (reg_offset.is_reg()) {
+      assert(reg_offset.type() == StorageType::INTEGER, "expected");
+      __ add(reg_oop_reg, reg_oop_reg, as_Register(reg_offset));
+    } else {
+      assert(reg_offset.is_stack(), "expected");
+      assert(reg_offset.stack_size() == 8, "expected long");
+      __ ld(r_tmp1, Address(fp, FP_BIAS + reg_offset.offset()));
+      __ add(reg_oop_reg, reg_oop_reg, r_tmp1);
+    }
+  } else {
+    assert(reg_oop.is_stack(), "expected");
+    assert(reg_oop.stack_size() == 8, "expected long");
+    assert(reg_offset.is_stack(), "expected");
+    assert(reg_offset.stack_size() == 8, "expected long");
+    __ ld(r_tmp1, Address(fp, FP_BIAS + reg_offset.offset()));
+    __ ld(r_tmp2, Address(fp, FP_BIAS + reg_oop.offset()));
+    __ add(r_tmp1, r_tmp2, r_tmp1);
+    __ sd(r_tmp1, Address(fp, FP_BIAS + reg_oop.offset()));
+  }
+}
+
+void DowncallLinker::StubGenerator::generate() {
   enum layout {
     fp_off,
     fp_off2,
@@ -147,23 +119,16 @@ void DowncallStubGenerator::generate() {
     // out arg area (e.g. for stack args)
   };
 
-  VMStorage shuffle_reg = as_VMStorage(x9);
+  GrowableArray<VMStorage> java_regs;
-  JavaCallingConvention in_conv;
+  ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs);
-  NativeCallingConvention out_conv(_input_registers);
+  bool has_objects = false;
-  ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg);
+  GrowableArray<VMStorage> filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature,
-
+                                                                                            _num_args, has_objects);
-#ifndef PRODUCT
+  assert(!(_needs_transition && has_objects), "can not pass objects when doing transition");
-  LogTarget(Trace, foreign, downcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
 
   int allocated_frame_size = 0;
   assert(_abi._shadow_space_bytes == 0, "not expecting shadow space on RISCV64");
-  allocated_frame_size += arg_shuffle.out_arg_bytes();
+  allocated_frame_size += ForeignGlobals::compute_out_arg_bytes(_input_registers);
 
   bool should_save_return_value = !_needs_return_buffer;
   RegSpiller out_reg_spiller(_output_registers);
@@ -190,6 +155,33 @@ void DowncallStubGenerator::generate() {
     allocated_frame_size += BytesPerWord;
   }
 
+  // The space we have allocated will look like:
+  //
+  // FP-> |                     |
+  //      |---------------------| = frame_bottom_offset = frame_size
+  //      | (optional)          |
+  //      | capture state buf   |
+  //      |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
+  //      | (optional)          |
+  //      | return buffer       |
+  //      |---------------------| = StubLocations::RETURN_BUFFER
+  // SP-> | out/stack args      | or | out_reg_spiller area |
+  //
+  // Note how the last chunk can be shared, since the 3 uses occur at different times.
+
+  VMStorage shuffle_reg = as_VMStorage(x9);
+  GrowableArray<VMStorage> out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
+  ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, downcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   allocated_frame_size = align_up(allocated_frame_size, 16);
   // _frame_size_slots is in 32-bit stack slots:
   _frame_size_slots += framesize + (allocated_frame_size >> LogBytesPerInt);
@@ -219,8 +211,12 @@ void DowncallStubGenerator::generate() {
     __ block_comment("} thread java2native");
   }
 
+  if (has_objects) {
+    add_offsets_to_oops(java_regs, as_VMStorage(t0), as_VMStorage(t1));
+  }
+
   __ block_comment("{ argument shuffle");
-  arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes, locs);
+  arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
   __ block_comment("} argument shuffle");
 
   __ jalr(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));

src/hotspot/cpu/riscv/foreignGlobals_riscv.cpp

@@ -163,21 +163,13 @@ static void move_fp(MacroAssembler* masm, int out_stk_bias,
   }
 }
 
-void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
   Register tmp_reg = as_Register(tmp);
   for (int i = 0; i < _moves.length(); i++) {
     Move move = _moves.at(i);
     VMStorage from_reg = move.from;
     VMStorage to_reg   = move.to;
 
-    // replace any placeholders
-    if (from_reg.type() == StorageType::PLACEHOLDER) {
-      from_reg = locs.get(from_reg);
-    }
-    if (to_reg.type() == StorageType::PLACEHOLDER) {
-      to_reg = locs.get(to_reg);
-    }
-
     switch (from_reg.type()) {
       case StorageType::INTEGER:
         assert(from_reg.segment_mask() == REG64_MASK, "only 64-bit integer register supported");

src/hotspot/cpu/riscv/upcallLinker_riscv.cpp

@@ -118,7 +118,6 @@ static const int upcall_stub_code_base_size = 1024;
 static const int upcall_stub_size_per_arg = 16;
 
 address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
-                                       BasicType* in_sig_bt, int total_in_args,
                                        BasicType* out_sig_bt, int total_out_args,
                                        BasicType ret_type,
                                        jobject jabi, jobject jconv,
@@ -127,26 +126,15 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   ResourceMark rm;
   const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi);
   const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv);
-  int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg);
+  int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg);
   CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1);
 
-  Register shuffle_reg = x9;
+  GrowableArray<VMStorage> unfiltered_out_regs;
-  JavaCallingConvention out_conv;
+  int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs);
-  NativeCallingConvention in_conv(call_regs._arg_regs);
-  ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, as_VMStorage(shuffle_reg));
   int preserved_bytes = SharedRuntime::out_preserve_stack_slots() * VMRegImpl::stack_slot_size;
-  int stack_bytes = preserved_bytes + arg_shuffle.out_arg_bytes();
+  int stack_bytes = preserved_bytes + out_arg_bytes;
   int out_arg_area = align_up(stack_bytes , StackAlignmentInBytes);
 
-#ifndef PRODUCT
-  LogTarget(Trace, foreign, upcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
-
   // out_arg_area (for stack arguments) doubles as shadow space for native calls.
   // make sure it is big enough.
   if (out_arg_area < frame::arg_reg_save_area_bytes) {
@@ -174,6 +162,20 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
     locs.set(StubLocations::RETURN_BUFFER, abi._scratch1);
   }
 
+  Register shuffle_reg = x9;
+  GrowableArray<VMStorage> in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs);
+  GrowableArray<VMStorage> filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs);
+  ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, as_VMStorage(shuffle_reg));
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, upcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   int frame_size = frame_bottom_offset;
   frame_size = align_up(frame_size, StackAlignmentInBytes);
 
@@ -252,7 +254,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
     __ la(as_Register(locs.get(StubLocations::RETURN_BUFFER)), Address(sp, ret_buf_offset));
   }
 
-  arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0, locs);
+  arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0);
   __ block_comment("} argument shuffle");
 
   __ block_comment("{ receiver ");

src/hotspot/cpu/s390/downcallLinker_s390.cpp

@@ -36,61 +36,6 @@
 
 #define __ _masm->
 
-class DowncallStubGenerator : public StubCodeGenerator {
-  BasicType* _signature;
-  int _num_args;
-  BasicType _ret_bt;
-  const ABIDescriptor& _abi;
-
-  const GrowableArray<VMStorage>&  _input_registers;
-  const GrowableArray<VMStorage>&  _output_registers;
-
-  bool _needs_return_buffer;
-  int _captured_state_mask;
-  bool _needs_transition;
-
-  int _frame_complete;
-  int _frame_size_slots;
-  OopMapSet* _oop_maps;
-  public:
-  DowncallStubGenerator(CodeBuffer* buffer,
-                         BasicType* signature,
-                         int num_args,
-                         BasicType ret_bt,
-                         const ABIDescriptor& abi,
-                         const GrowableArray<VMStorage>& input_registers,
-                         const GrowableArray<VMStorage>& output_registers,
-                         bool needs_return_buffer,
-                         int captured_state_mask,
-                         bool needs_transition)
-    :StubCodeGenerator(buffer, PrintMethodHandleStubs),
-    _signature(signature),
-    _num_args(num_args),
-    _ret_bt(ret_bt),
-    _abi(abi),
-    _input_registers(input_registers),
-    _output_registers(output_registers),
-    _needs_return_buffer(needs_return_buffer),
-    _captured_state_mask(captured_state_mask),
-    _needs_transition(needs_transition),
-    _frame_complete(0),
-    _frame_size_slots(0),
-    _oop_maps(nullptr) {
-    }
-  void generate();
-  int frame_complete() const {
-    return _frame_complete;
-  }
-
-  int framesize() const {
-    return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt));
-  }
-
-  OopMapSet* oop_maps() const {
-    return _oop_maps;
-  }
-};
-
 static const int native_invoker_code_base_size = 512;
 static const int native_invoker_size_per_args = 8;
 
@@ -108,10 +53,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   int locs_size = 1; //must be non zero
   CodeBuffer code("nep_invoker_blob", code_size, locs_size);
 
-  DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi,
+  StubGenerator g(&code, signature, num_args, ret_bt, abi,
-                          input_registers, output_registers,
+                  input_registers, output_registers,
-                          needs_return_buffer, captured_state_mask,
+                  needs_return_buffer, captured_state_mask,
-                          needs_transition);
+                  needs_transition);
   g.generate();
   code.log_section_sizes("nep_invoker_blob");
 
@@ -134,28 +79,45 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   return stub;
 }
 
-void DowncallStubGenerator::generate() {
+static constexpr int FP_BIAS = frame::z_jit_out_preserve_size;
+
+void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset,
+                                                         VMStorage tmp1, VMStorage tmp2) const {
+  Register r_tmp1 = as_Register(tmp1);
+  Register r_tmp2 = as_Register(tmp2);
+  Register callerSP = Z_R11;
+  if (reg_oop.is_reg()) {
+    assert(reg_oop.type() == StorageType::INTEGER, "expected");
+    Register reg_oop_reg = as_Register(reg_oop);
+    if (reg_offset.is_reg()) {
+      assert(reg_offset.type() == StorageType::INTEGER, "expected");
+      __ z_agr(reg_oop_reg, as_Register(reg_offset));
+    } else {
+      assert(reg_offset.is_stack(), "expected");
+      assert(reg_offset.stack_size() == 8, "expected long");
+      Address offset_addr(callerSP, FP_BIAS + reg_offset.offset());
+      __ z_ag(reg_oop_reg, offset_addr);
+    }
+  } else {
+    assert(reg_oop.is_stack(), "expected");
+    assert(reg_oop.stack_size() == 8, "expected long");
+    assert(reg_offset.is_stack(), "expected");
+    assert(reg_offset.stack_size() == 8, "expected long");
+    Address offset_addr(callerSP, FP_BIAS + reg_offset.offset());
+    Address oop_addr(callerSP, FP_BIAS + reg_oop.offset());
+    __ mem2reg_opt(r_tmp2, oop_addr, true);
+    __ z_ag(r_tmp2, offset_addr);
+    __ reg2mem_opt(r_tmp2, oop_addr, true);
+  }
+}
+
+void DowncallLinker::StubGenerator::generate() {
   Register call_target_address = Z_R1_scratch,
            tmp = Z_R0_scratch;
 
-  VMStorage shuffle_reg = _abi._scratch1;
-
-  JavaCallingConvention in_conv;
-  NativeCallingConvention out_conv(_input_registers);
-  ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg);
-
-#ifndef PRODUCT
-  LogTarget(Trace, foreign, downcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
-
   assert(_abi._shadow_space_bytes == frame::z_abi_160_size, "expected space according to ABI");
   int allocated_frame_size = _abi._shadow_space_bytes;
-  allocated_frame_size += arg_shuffle.out_arg_bytes();
+  allocated_frame_size += ForeignGlobals::compute_out_arg_bytes(_input_registers);
 
   assert(!_needs_return_buffer, "unexpected needs_return_buffer");
   RegSpiller out_reg_spiller(_output_registers);
@@ -172,6 +134,26 @@ void DowncallStubGenerator::generate() {
     __ block_comment("} _captured_state_mask is set");
   }
 
+  VMStorage shuffle_reg = _abi._scratch1;
+  GrowableArray<VMStorage> java_regs;
+  ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs);
+  bool has_objects = false;
+  GrowableArray<VMStorage> filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature,
+                                                                                            _num_args, has_objects);
+  assert(!(_needs_transition && has_objects), "can not pass objects when doing transition");
+
+  GrowableArray<VMStorage> out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
+  ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, _abi._scratch1);
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, downcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   allocated_frame_size = align_up(allocated_frame_size, StackAlignmentInBytes);
   _frame_size_slots = allocated_frame_size >> LogBytesPerInt;
 
@@ -196,8 +178,11 @@ void DowncallStubGenerator::generate() {
     __ set_thread_state(_thread_in_native);
     __ block_comment("} thread java2native");
   }
+  if (has_objects) {
+    add_offsets_to_oops(java_regs, _abi._scratch1, _abi._scratch2);
+  }
   __ block_comment("{ argument shuffle");
-  arg_shuffle.generate(_masm, shuffle_reg, frame::z_jit_out_preserve_size, _abi._shadow_space_bytes, locs);
+  arg_shuffle.generate(_masm, shuffle_reg, frame::z_jit_out_preserve_size, _abi._shadow_space_bytes);
   __ block_comment("} argument shuffle");
 
   __ call(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));

src/hotspot/cpu/s390/foreignGlobals_s390.cpp

@@ -200,21 +200,13 @@ static void move_stack(MacroAssembler* masm, Register tmp_reg, int in_stk_bias,
   }
 }
 
-void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
   Register tmp_reg = as_Register(tmp);
   for (int i = 0; i < _moves.length(); i++) {
     Move move = _moves.at(i);
     VMStorage from_reg = move.from;
     VMStorage to_reg   = move.to;
 
-    // replace any placeholders
-    if (from_reg.type() == StorageType::PLACEHOLDER) {
-      from_reg = locs.get(from_reg);
-    }
-    if (to_reg.type() == StorageType::PLACEHOLDER) {
-      to_reg = locs.get(to_reg);
-    }
-
     switch (from_reg.type()) {
       case StorageType::INTEGER:
         move_reg(masm, out_stk_bias, from_reg, to_reg);

src/hotspot/cpu/s390/upcallLinker_s390.cpp

@@ -117,7 +117,6 @@ static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescr
 static const int upcall_stub_code_base_size = 1024;
 static const int upcall_stub_size_per_arg = 16; // arg save & restore + move
 address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
-                                       BasicType* in_sig_bt, int total_in_args,
                                        BasicType* out_sig_bt, int total_out_args,
                                        BasicType ret_type,
                                        jobject jabi, jobject jconv,
@@ -125,28 +124,16 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   ResourceMark rm;
   const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi);
   const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv);
-  int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg);
+  int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg);
   CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 0);
 
   Register call_target_address = Z_R1_scratch;
 
-  VMStorage shuffle_reg = abi._scratch1;
+  GrowableArray<VMStorage> unfiltered_out_regs;
-  JavaCallingConvention out_conv;
+  int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs);
-  NativeCallingConvention in_conv(call_regs._arg_regs);
-  ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, shuffle_reg);
 
   // The Java call uses the JIT ABI, but we also call C.
-  int out_arg_area = MAX2(frame::z_jit_out_preserve_size + arg_shuffle.out_arg_bytes(), (int)frame::z_abi_160_size);
+  int out_arg_area = MAX2(frame::z_jit_out_preserve_size + out_arg_bytes, (int)frame::z_abi_160_size);
-
-#ifndef PRODUCT
-  LogTarget(Trace, foreign, upcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
-
 
   int reg_save_area_size = compute_reg_save_area_size(abi);
   RegSpiller arg_spiller(call_regs._arg_regs);
@@ -158,8 +145,22 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   int frame_data_offset     = reg_save_area_offset  + reg_save_area_size;
   int frame_bottom_offset   = frame_data_offset     + sizeof(UpcallStub::FrameData);
 
-  int frame_size = align_up(frame_bottom_offset, StackAlignmentInBytes);
   StubLocations locs;
+  VMStorage shuffle_reg = abi._scratch1;
+  GrowableArray<VMStorage> in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs);
+  GrowableArray<VMStorage> filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs);
+  ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, shuffle_reg);
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, upcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
+  int frame_size = align_up(frame_bottom_offset, StackAlignmentInBytes);
 
   // The space we have allocated will look like:
   //
@@ -209,7 +210,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
 
   arg_spiller.generate_fill(_masm, arg_save_area_offset);
   __ block_comment("{ argument shuffle");
-  arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, frame::z_jit_out_preserve_size, locs);
+  arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, frame::z_jit_out_preserve_size);
   __ block_comment("} argument shuffle");
 
   __ block_comment("{ receiver ");

src/hotspot/cpu/x86/downcallLinker_x86_32.cpp

@@ -36,3 +36,8 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   Unimplemented();
   return nullptr;
 }
+
+void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset,
+                                                         VMStorage tmp1, VMStorage tmp2) const {
+  Unimplemented();
+}

src/hotspot/cpu/x86/downcallLinker_x86_64.cpp

@@ -30,68 +30,10 @@
 #include "prims/downcallLinker.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/sharedRuntime.hpp"
-#include "runtime/stubCodeGenerator.hpp"
 #include "utilities/formatBuffer.hpp"
 
 #define __ _masm->
 
-class DowncallStubGenerator : public StubCodeGenerator {
-  BasicType* _signature;
-  int _num_args;
-  BasicType _ret_bt;
-
-  const ABIDescriptor& _abi;
-  const GrowableArray<VMStorage>& _input_registers;
-  const GrowableArray<VMStorage>& _output_registers;
-
-  bool _needs_return_buffer;
-  int _captured_state_mask;
-  bool _needs_transition;
-
-  int _frame_complete;
-  int _frame_size_slots;
-  OopMapSet* _oop_maps;
-public:
-  DowncallStubGenerator(CodeBuffer* buffer,
-                         BasicType* signature,
-                         int num_args,
-                         BasicType ret_bt,
-                         const ABIDescriptor& abi,
-                         const GrowableArray<VMStorage>& input_registers,
-                         const GrowableArray<VMStorage>& output_registers,
-                         bool needs_return_buffer,
-                         int captured_state_mask,
-                         bool needs_transition)
-   : StubCodeGenerator(buffer, PrintMethodHandleStubs),
-     _signature(signature),
-     _num_args(num_args),
-     _ret_bt(ret_bt),
-     _abi(abi),
-     _input_registers(input_registers),
-     _output_registers(output_registers),
-     _needs_return_buffer(needs_return_buffer),
-     _captured_state_mask(captured_state_mask),
-     _needs_transition(needs_transition),
-     _frame_complete(0),
-     _frame_size_slots(0),
-     _oop_maps(nullptr) {
-  }
-
-  void generate();
-
-  int frame_complete() const {
-    return _frame_complete;
-  }
-
-  int framesize() const { // frame size in 64-bit words
-    return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt));
-  }
-
-  OopMapSet* oop_maps() const {
-    return _oop_maps;
-  }
-};
-
 static const int native_invoker_code_base_size = 512;
 static const int native_invoker_size_per_arg = 8;
 
@@ -107,10 +49,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   int code_size = native_invoker_code_base_size + (num_args * native_invoker_size_per_arg);
   int locs_size = 1; // can not be zero
   CodeBuffer code("nep_invoker_blob", code_size, locs_size);
-  DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi,
+  StubGenerator g(&code, signature, num_args, ret_bt, abi,
-                          input_registers, output_registers,
+                  input_registers, output_registers,
-                          needs_return_buffer, captured_state_mask,
+                  needs_return_buffer, captured_state_mask,
-                          needs_transition);
+                  needs_transition);
   g.generate();
   code.log_section_sizes("nep_invoker_blob");
 
@@ -133,7 +75,41 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   return stub;
 }
 
-void DowncallStubGenerator::generate() {
+static constexpr int RBP_BIAS = 16; // skip old rbp and return address
+
+void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset,
+                                                         VMStorage tmp1, VMStorage tmp2) const {
+  if (reg_oop.is_reg()) {
+    assert(reg_oop.type() == StorageType::INTEGER, "expected");
+    if (reg_offset.is_reg()) {
+      assert(reg_offset.type() == StorageType::INTEGER, "expected");
+      __ addptr(as_Register(reg_oop), as_Register(reg_offset));
+    } else {
+      assert(reg_offset.is_stack(), "expected");
+      Address offset_addr(rbp, RBP_BIAS + reg_offset.offset());
+      __ addptr(as_Register(reg_oop), offset_addr);
+    }
+  } else {
+    assert(reg_oop.is_stack(), "expected");
+    assert(reg_offset.is_stack(), "expected");
+    Address offset_addr(rbp, RBP_BIAS + reg_offset.offset());
+    Address oop_addr(rbp, RBP_BIAS + reg_oop.offset());
+    __ movptr(as_Register(tmp1), offset_addr);
+    __ addptr(oop_addr, as_Register(tmp1));
+  }
+}
+
+static void runtime_call(MacroAssembler* _masm, address target) {
+  __ vzeroupper();
+  __ mov(r12, rsp); // remember sp
+  __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
+  __ andptr(rsp, -16); // align stack as required by ABI
+  __ call(RuntimeAddress(target));
+  __ mov(rsp, r12); // restore sp
+  __ reinit_heapbase();
+}
+
+void DowncallLinker::StubGenerator::generate() {
   enum layout {
     rbp_off,
     rbp_off2,
@@ -146,24 +122,17 @@ void DowncallStubGenerator::generate() {
     // out arg area (e.g. for stack args)
   };
 
-  VMStorage shuffle_reg = as_VMStorage(rbx);
+  GrowableArray<VMStorage> java_regs;
-  JavaCallingConvention in_conv;
+  ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs);
-  NativeCallingConvention out_conv(_input_registers);
+  bool has_objects = false;
-  ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg);
+  GrowableArray<VMStorage> filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature,
-
+                                                                                             _num_args, has_objects);
-#ifndef PRODUCT
+  assert(!(_needs_transition && has_objects), "can not pass objects when doing transition");
-  LogTarget(Trace, foreign, downcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
 
   // in bytes
   int allocated_frame_size = 0;
   allocated_frame_size += _abi._shadow_space_bytes;
-  allocated_frame_size += arg_shuffle.out_arg_bytes();
+  allocated_frame_size += ForeignGlobals::compute_out_arg_bytes(_input_registers);
 
   // when we don't use a return buffer we need to spill the return value around our slow path calls
   bool should_save_return_value = !_needs_return_buffer;
@@ -191,6 +160,35 @@ void DowncallStubGenerator::generate() {
     allocated_frame_size += BytesPerWord;
   }
 
+  // The space we have allocated will look like:
+  //
+  // FP-> |                     |
+  //      |---------------------| = frame_bottom_offset = frame_size
+  //      | (optional)          |
+  //      | capture state buf   |
+  //      |---------------------| = StubLocations::CAPTURED_STATE_BUFFER
+  //      | (optional)          |
+  //      | return buffer       |
+  //      |---------------------| = StubLocations::RETURN_BUFFER
+  //      | out/stack args      |    |                      |
+  //      |---------------------| or | ret. val. spill area |
+  // SP-> | shadow space        |    |                      |
+  //
+  // Note how the last chunk can be shared, since the 3 uses occur at different times.
+
+  GrowableArray<VMStorage> out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs);
+  VMStorage shuffle_reg = as_VMStorage(rbx);
+  ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg);
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, downcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   allocated_frame_size = align_up(allocated_frame_size, 16);
   _frame_size_slots += framesize_base + (allocated_frame_size >> LogBytesPerInt);
   assert(is_even(_frame_size_slots/2), "sp not 16-byte aligned");
@@ -219,12 +217,16 @@ void DowncallStubGenerator::generate() {
     __ block_comment("} thread java2native");
   }
 
+  if (has_objects) {
+    add_offsets_to_oops(java_regs, as_VMStorage(rscratch1), VMStorage::invalid());
+  }
+
   __ block_comment("{ argument shuffle");
-  arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes, locs);
+  arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes);
   __ block_comment("} argument shuffle");
 
   __ call(as_Register(locs.get(StubLocations::TARGET_ADDRESS)));
-  // this call is assumed not to have killed r15_thread
+  assert(!_abi.is_volatile_reg(r15_thread), "Call assumed not to kill r15");
 
   if (_needs_return_buffer) {
     __ movptr(rscratch1, Address(rsp, locs.data_offset(StubLocations::RETURN_BUFFER)));
@@ -247,7 +249,6 @@ void DowncallStubGenerator::generate() {
 
   if (_captured_state_mask != 0) {
     __ block_comment("{ save thread local");
-    __ vzeroupper();
 
     if (should_save_return_value) {
       out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
@@ -255,12 +256,7 @@ void DowncallStubGenerator::generate() {
 
     __ movptr(c_rarg0, Address(rsp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER)));
     __ movl(c_rarg1, _captured_state_mask);
-    __ mov(r12, rsp); // remember sp
+    runtime_call(_masm, CAST_FROM_FN_PTR(address, DowncallLinker::capture_state));
-    __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
-    __ andptr(rsp, -16); // align stack as required by ABI
-    __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state)));
-    __ mov(rsp, r12); // restore sp
-    __ reinit_heapbase();
 
     if (should_save_return_value) {
       out_reg_spiller.generate_fill(_masm, spill_rsp_offset);
@@ -314,19 +310,13 @@ void DowncallStubGenerator::generate() {
   if (_needs_transition) {
     __ block_comment("{ L_safepoint_poll_slow_path");
     __ bind(L_safepoint_poll_slow_path);
-    __ vzeroupper();
 
     if (should_save_return_value) {
       out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
     }
 
     __ mov(c_rarg0, r15_thread);
-    __ mov(r12, rsp); // remember sp
+    runtime_call(_masm, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
-    __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
-    __ andptr(rsp, -16); // align stack as required by ABI
-    __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
-    __ mov(rsp, r12); // restore sp
-    __ reinit_heapbase();
 
     if (should_save_return_value) {
       out_reg_spiller.generate_fill(_masm, spill_rsp_offset);
@@ -339,18 +329,12 @@ void DowncallStubGenerator::generate() {
 
     __ block_comment("{ L_reguard");
     __ bind(L_reguard);
-    __ vzeroupper();
 
     if (should_save_return_value) {
       out_reg_spiller.generate_spill(_masm, spill_rsp_offset);
     }
 
-    __ mov(r12, rsp); // remember sp
+    runtime_call(_masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
-    __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
-    __ andptr(rsp, -16); // align stack as required by ABI
-    __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
-    __ mov(rsp, r12); // restore sp
-    __ reinit_heapbase();
 
     if (should_save_return_value) {
       out_reg_spiller.generate_fill(_masm, spill_rsp_offset);

src/hotspot/cpu/x86/foreignGlobals_x86_32.cpp

@@ -50,6 +50,6 @@ void RegSpiller::pd_load_reg(MacroAssembler* masm, int offset, VMStorage reg) {
   Unimplemented();
 }
 
-void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
   Unimplemented();
 }

src/hotspot/cpu/x86/foreignGlobals_x86_64.cpp

@@ -162,21 +162,13 @@ static void move_xmm(MacroAssembler* masm, int out_stk_bias,
   }
 }
 
-void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
   Register tmp_reg = as_Register(tmp);
   for (int i = 0; i < _moves.length(); i++) {
     Move move = _moves.at(i);
     VMStorage from_reg = move.from;
     VMStorage to_reg   = move.to;
 
-    // replace any placeholders
-    if (from_reg.type() == StorageType::PLACEHOLDER) {
-      from_reg = locs.get(from_reg);
-    }
-    if (to_reg.type() == StorageType::PLACEHOLDER) {
-      to_reg = locs.get(to_reg);
-    }
-
     switch (from_reg.type()) {
       case StorageType::INTEGER:
         assert(from_reg.segment_mask() == REG64_MASK, "only 64-bit register supported");

src/hotspot/cpu/x86/upcallLinker_x86_32.cpp

@@ -25,7 +25,6 @@
 #include "prims/upcallLinker.hpp"
 
 address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
-                                       BasicType* in_sig_bt, int total_in_args,
                                        BasicType* out_sig_bt, int total_out_args,
                                        BasicType ret_type,
                                        jobject jabi, jobject jconv,

src/hotspot/cpu/x86/upcallLinker_x86_64.cpp

@@ -169,33 +169,21 @@ static const int upcall_stub_code_base_size = 1024;
 static const int upcall_stub_size_per_arg = 16;
 
 address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
-                                       BasicType* in_sig_bt, int total_in_args,
                                        BasicType* out_sig_bt, int total_out_args,
                                        BasicType ret_type,
                                        jobject jabi, jobject jconv,
                                        bool needs_return_buffer, int ret_buf_size) {
   const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi);
   const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv);
-  int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg);
+  int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg);
   CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1);
 
-  VMStorage shuffle_reg = as_VMStorage(rbx);
+  GrowableArray<VMStorage> unfiltered_out_regs;
-  JavaCallingConvention out_conv;
+  int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs);
-  NativeCallingConvention in_conv(call_regs._arg_regs);
-  ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, shuffle_reg);
   int preserved_bytes = SharedRuntime::out_preserve_stack_slots() * VMRegImpl::stack_slot_size;
-  int stack_bytes = preserved_bytes + arg_shuffle.out_arg_bytes();
+  int stack_bytes = preserved_bytes + out_arg_bytes;
   int out_arg_area = align_up(stack_bytes , StackAlignmentInBytes);
 
-#ifndef PRODUCT
-  LogTarget(Trace, foreign, upcall) lt;
-  if (lt.is_enabled()) {
-    ResourceMark rm;
-    LogStream ls(lt);
-    arg_shuffle.print_on(&ls);
-  }
-#endif
-
   // out_arg_area (for stack arguments) doubles as shadow space for native calls.
   // make sure it is big enough.
   if (out_arg_area < frame::arg_reg_save_area_bytes) {
@@ -203,13 +191,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   }
 
   int reg_save_area_size = compute_reg_save_area_size(abi);
-  RegSpiller arg_spilller(call_regs._arg_regs);
+  RegSpiller arg_spiller(call_regs._arg_regs);
   RegSpiller result_spiller(call_regs._ret_regs);
 
   int shuffle_area_offset    = 0;
   int res_save_area_offset   = shuffle_area_offset    + out_arg_area;
   int arg_save_area_offset   = res_save_area_offset   + result_spiller.spill_size_bytes();
-  int reg_save_area_offset   = arg_save_area_offset   + arg_spilller.spill_size_bytes();
+  int reg_save_area_offset   = arg_save_area_offset   + arg_spiller.spill_size_bytes();
   int frame_data_offset      = reg_save_area_offset   + reg_save_area_size;
   int frame_bottom_offset    = frame_data_offset      + sizeof(UpcallStub::FrameData);
 
@@ -223,6 +211,20 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
     locs.set(StubLocations::RETURN_BUFFER, abi._scratch1);
   }
 
+  VMStorage shuffle_reg = as_VMStorage(rbx);
+  GrowableArray<VMStorage> in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs);
+  GrowableArray<VMStorage> filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs);
+  ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, shuffle_reg);
+
+#ifndef PRODUCT
+  LogTarget(Trace, foreign, upcall) lt;
+  if (lt.is_enabled()) {
+    ResourceMark rm;
+    LogStream ls(lt);
+    arg_shuffle.print_on(&ls);
+  }
+#endif
+
   int frame_size = frame_bottom_offset;
   frame_size = align_up(frame_size, StackAlignmentInBytes);
 
@@ -265,7 +267,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
 
   // we have to always spill args since we need to do a call to get the thread
   // (and maybe attach it).
-  arg_spilller.generate_spill(_masm, arg_save_area_offset);
+  arg_spiller.generate_spill(_masm, arg_save_area_offset);
 
   preserve_callee_saved_registers(_masm, abi, reg_save_area_offset);
 
@@ -280,12 +282,12 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
   __ block_comment("} on_entry");
 
   __ block_comment("{ argument shuffle");
-  arg_spilller.generate_fill(_masm, arg_save_area_offset);
+  arg_spiller.generate_fill(_masm, arg_save_area_offset);
   if (needs_return_buffer) {
     assert(ret_buf_offset != -1, "no return buffer allocated");
     __ lea(as_Register(locs.get(StubLocations::RETURN_BUFFER)), Address(rsp, ret_buf_offset));
   }
-  arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, 0, locs);
+  arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, 0);
   __ block_comment("} argument shuffle");
 
   __ block_comment("{ receiver ");

src/hotspot/cpu/zero/downcallLinker_zero.cpp

@@ -36,3 +36,8 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature,
   Unimplemented();
   return nullptr;
 }
+
+void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset,
+                                                         VMStorage tmp1, VMStorage tmp2) const {
+  Unimplemented();
+}

src/hotspot/cpu/zero/foreignGlobals_zero.cpp

@@ -50,6 +50,6 @@ void RegSpiller::pd_load_reg(MacroAssembler* masm, int offset, VMStorage reg) {
   Unimplemented();
 }
 
-void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
   Unimplemented();
 }

src/hotspot/cpu/zero/upcallLinker_zero.cpp

@@ -24,8 +24,7 @@
 #include "precompiled.hpp"
 #include "prims/upcallLinker.hpp"
 
-address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
+address UpcallLinker::make_upcall_stub(jobject mh, Method* entry,
-                                       BasicType* in_sig_bt, int total_in_args,
                                        BasicType* out_sig_bt, int total_out_args,
                                        BasicType ret_type,
                                        jobject jabi, jobject jconv,

src/hotspot/share/prims/downcallLinker.cpp

@@ -23,6 +23,8 @@
 
 #include "precompiled.hpp"
 #include "downcallLinker.hpp"
+#include "gc/shared/gcLocker.inline.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
 
 #include <cerrno>
 #ifdef _WIN64
@@ -30,7 +32,8 @@
 #include <Winsock2.h>
 #endif
 
-void DowncallLinker::capture_state(int32_t* value_ptr, int captured_state_mask) {
+// We call this from _thread_in_native, right after a downcall
+JVM_LEAF(void, DowncallLinker::capture_state(int32_t* value_ptr, int captured_state_mask))
   // keep in synch with jdk.internal.foreign.abi.PreservableValues
   enum PreservableValues {
     NONE = 0,
@@ -51,4 +54,19 @@ void DowncallLinker::capture_state(int32_t* value_ptr, int captured_state_mask)
   if (captured_state_mask & ERRNO) {
     *value_ptr = errno;
   }
+JVM_END
+
+void DowncallLinker::StubGenerator::add_offsets_to_oops(GrowableArray<VMStorage>& java_regs, VMStorage tmp1, VMStorage tmp2) const {
+  int reg_idx = 0;
+  for (int sig_idx = 0; sig_idx < _num_args; sig_idx++) {
+    if (_signature[sig_idx] == T_OBJECT) {
+      assert(_signature[sig_idx + 1] == T_LONG, "expected offset after oop");
+      VMStorage reg_oop = java_regs.at(reg_idx++);
+      VMStorage reg_offset = java_regs.at(reg_idx++);
+      sig_idx++; // skip offset
+      pd_add_offset_to_oop(reg_oop, reg_offset, tmp1, tmp2);
+    } else if (_signature[sig_idx] != T_VOID) {
+      reg_idx++;
+    }
+  }
 }

src/hotspot/share/prims/downcallLinker.hpp

@@ -25,6 +25,7 @@
 #define SHARE_VM_PRIMS_DOWNCALLLINKER_HPP
 
 #include "prims/foreignGlobals.hpp"
+#include "runtime/stubCodeGenerator.hpp"
 
 class RuntimeStub;
 
@@ -41,6 +42,60 @@ public:
                                          bool needs_transition);
 
   static void capture_state(int32_t* value_ptr, int captured_state_mask);
+
+  class StubGenerator : public StubCodeGenerator {
+    BasicType* _signature;
+    int _num_args;
+    BasicType _ret_bt;
+
+    const ABIDescriptor& _abi;
+    const GrowableArray<VMStorage>& _input_registers;
+    const GrowableArray<VMStorage>& _output_registers;
+
+    bool _needs_return_buffer;
+    int _captured_state_mask;
+    bool _needs_transition;
+
+    int _frame_complete;
+    int _frame_size_slots;
+    OopMapSet* _oop_maps;
+  public:
+    StubGenerator(CodeBuffer* buffer,
+                  BasicType* signature,
+                  int num_args,
+                  BasicType ret_bt,
+                  const ABIDescriptor& abi,
+                  const GrowableArray<VMStorage>& input_registers,
+                  const GrowableArray<VMStorage>& output_registers,
+                  bool needs_return_buffer,
+                  int captured_state_mask,
+                  bool needs_transition)
+    : StubCodeGenerator(buffer, PrintMethodHandleStubs),
+      _signature(signature),
+      _num_args(num_args),
+      _ret_bt(ret_bt),
+      _abi(abi),
+      _input_registers(input_registers),
+      _output_registers(output_registers),
+      _needs_return_buffer(needs_return_buffer),
+      _captured_state_mask(captured_state_mask),
+      _needs_transition(needs_transition),
+      _frame_complete(0),
+      _frame_size_slots(0),
+      _oop_maps(nullptr) {
+    }
+
+    void generate();
+
+    int frame_complete() const { return _frame_complete; }
+    // frame size in words
+    int framesize() const { return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt)); }
+
+    OopMapSet* oop_maps() const { return _oop_maps; }
+
+    void pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, VMStorage tmp1, VMStorage tmp2) const;
+    void add_offsets_to_oops(GrowableArray<VMStorage>& java_regs, VMStorage tmp1, VMStorage tmp2) const;
+  };
 };
 
 #endif // SHARE_VM_PRIMS_DOWNCALLLINKER_HPP

src/hotspot/share/prims/foreignGlobals.cpp

@@ -126,62 +126,70 @@ void ArgumentShuffle::print_on(outputStream* os) const {
     to_reg.print_on(os);
     os->print_cr("");
   }
-  os->print_cr("Stack argument bytes: %d", _out_arg_bytes);
   os->print_cr("}");
 }
 
-int NativeCallingConvention::calling_convention(const BasicType* sig_bt, VMStorage* out_regs, int num_args) const {
+int ForeignGlobals::compute_out_arg_bytes(const GrowableArray<VMStorage>& out_regs) {
-  int src_pos = 0;
   uint32_t max_stack_offset = 0;
-  for (int i = 0; i < num_args; i++) {
+  for (VMStorage reg : out_regs) {
-    switch (sig_bt[i]) {
+    if (reg.is_stack())
-      case T_BOOLEAN:
+      max_stack_offset = MAX2(max_stack_offset, reg.offset() + reg.stack_size());
-      case T_CHAR:
-      case T_BYTE:
-      case T_SHORT:
-      case T_INT:
-      case T_FLOAT: {
-        VMStorage reg = _input_regs.at(src_pos++);
-        out_regs[i] = reg;
-        if (reg.is_stack())
-          max_stack_offset = MAX2(max_stack_offset, reg.offset() + reg.stack_size());
-        break;
-      }
-      case T_LONG:
-      case T_DOUBLE: {
-        assert((i + 1) < num_args && sig_bt[i + 1] == T_VOID, "expecting half");
-        VMStorage reg = _input_regs.at(src_pos++);
-        out_regs[i] = reg;
-        if (reg.is_stack())
-          max_stack_offset = MAX2(max_stack_offset, reg.offset() + reg.stack_size());
-        break;
-      }
-      case T_VOID: // Halves of longs and doubles
-        assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
-        out_regs[i] = VMStorage::invalid();
-        break;
-      default:
-        ShouldNotReachHere();
-        break;
-    }
   }
   return align_up(max_stack_offset, 8);
 }
 
-int JavaCallingConvention::calling_convention(const BasicType* sig_bt, VMStorage* regs, int num_args) const {
+int ForeignGlobals::java_calling_convention(const BasicType* signature, int num_args, GrowableArray<VMStorage>& out_regs) {
   VMRegPair* vm_regs = NEW_RESOURCE_ARRAY(VMRegPair, num_args);
-  int slots = SharedRuntime::java_calling_convention(sig_bt, vm_regs, num_args);
+  int slots = SharedRuntime::java_calling_convention(signature, vm_regs, num_args);
   for (int i = 0; i < num_args; i++) {
     VMRegPair pair = vm_regs[i];
     // note, we ignore second here. Signature should consist of register-size values. So there should be
     // no need for multi-register pairs.
-    //assert(!pair.first()->is_valid() || pair.is_single_reg(), "must be: %s");
+    if (signature[i] != T_VOID) {
-    regs[i] = as_VMStorage(pair.first(), sig_bt[i]);
+      out_regs.push(as_VMStorage(pair.first(), signature[i]));
+    }
   }
   return slots << LogBytesPerInt;
 }
 
-class ComputeMoveOrder: public StackObj {
+GrowableArray<VMStorage> ForeignGlobals::replace_place_holders(const GrowableArray<VMStorage>& regs, const StubLocations& locs) {
+  GrowableArray<VMStorage> result(regs.length());
+  for (VMStorage reg : regs) {
+    result.push(reg.type() == StorageType::PLACEHOLDER ? locs.get(reg) : reg);
+  }
+  return result;
+}
+
+GrowableArray<VMStorage> ForeignGlobals::upcall_filter_receiver_reg(const GrowableArray<VMStorage>& unfiltered_regs) {
+  GrowableArray<VMStorage> out(unfiltered_regs.length() - 1);
+  // drop first arg reg
+  for (int i = 1; i < unfiltered_regs.length(); i++) {
+    out.push(unfiltered_regs.at(i));
+  }
+  return out;
+}
+
+GrowableArray<VMStorage> ForeignGlobals::downcall_filter_offset_regs(const GrowableArray<VMStorage>& regs,
+                                                                     BasicType* signature, int num_args,
+                                                                     bool& has_objects) {
+  GrowableArray<VMStorage> result(regs.length());
+  int reg_idx = 0;
+  for (int sig_idx = 0; sig_idx < num_args; sig_idx++) {
+    if (signature[sig_idx] == T_VOID) {
+      continue; // ignore upper halves
+    }
+
+    result.push(regs.at(reg_idx++));
+    if (signature[sig_idx] == T_OBJECT) {
+      has_objects = true;
+      sig_idx++; // skip offset
+      reg_idx++;
+    }
+  }
+  return result;
+}
+
+class ArgumentShuffle::ComputeMoveOrder: public StackObj {
   class MoveOperation;
 
   // segment_mask_or_size is not taken into account since
@@ -258,42 +266,29 @@ class ComputeMoveOrder: public StackObj {
   };
 
  private:
-  int _total_in_args;
+  const GrowableArray<VMStorage>& _in_regs;
-  const VMStorage* _in_regs;
+  const GrowableArray<VMStorage>& _out_regs;
-  int _total_out_args;
-  const VMStorage* _out_regs;
-  const BasicType* _in_sig_bt;
   VMStorage _tmp_vmreg;
   GrowableArray<MoveOperation*> _edges;
   GrowableArray<Move> _moves;
 
  public:
-  ComputeMoveOrder(int total_in_args, const VMStorage* in_regs, int total_out_args, VMStorage* out_regs,
+  ComputeMoveOrder(const GrowableArray<VMStorage>& in_regs,
-                   const BasicType* in_sig_bt, VMStorage tmp_vmreg) :
+                   const GrowableArray<VMStorage>& out_regs,
-      _total_in_args(total_in_args),
+                   VMStorage tmp_vmreg) :
       _in_regs(in_regs),
-      _total_out_args(total_out_args),
       _out_regs(out_regs),
-      _in_sig_bt(in_sig_bt),
       _tmp_vmreg(tmp_vmreg),
-      _edges(total_in_args),
+      _edges(in_regs.length()),
-      _moves(total_in_args) {
+      _moves(in_regs.length()) {
+    assert(in_regs.length() == out_regs.length(),
+      "stray registers? %d != %d", in_regs.length(), out_regs.length());
   }
 
   void compute() {
-    assert(_total_out_args >= _total_in_args, "can only add prefix args");
+    for (int i = 0; i < _in_regs.length(); i++) {
-    // Note that total_out_args args can be greater than total_in_args in the case of upcalls.
+      VMStorage in_reg = _in_regs.at(i);
-    // There will be a leading MH receiver arg in the out args in that case.
+      VMStorage out_reg = _out_regs.at(i);
-    //
-    // Leading args in the out args will be ignored below because we iterate from the end of
-    // the register arrays until !(in_idx >= 0), and total_in_args is smaller.
-    //
-    // Stub code adds a move for the receiver to j_rarg0 (and potential other prefix args) manually.
-    for (int in_idx = _total_in_args - 1, out_idx = _total_out_args - 1; in_idx >= 0; in_idx--, out_idx--) {
-      BasicType bt = _in_sig_bt[in_idx];
-      assert(bt != T_ARRAY, "array not expected");
-      VMStorage in_reg = _in_regs[in_idx];
-      VMStorage out_reg = _out_regs[out_idx];
 
       if (out_reg.is_stack() || out_reg.is_frame_data()) {
         // Move operations where the dest is the stack can all be
@@ -301,12 +296,8 @@ class ComputeMoveOrder: public StackObj {
         // The input and output stack spaces are distinct from each other.
         Move move{in_reg, out_reg};
         _moves.push(move);
-      } else if (in_reg == out_reg
+      } else if (in_reg == out_reg) {
-                 || bt == T_VOID) {
+        // Can skip non-stack identity moves.
-        // 1. Can skip non-stack identity moves.
-        //
-        // 2. Upper half of long or double (T_VOID).
-        //    Don't need to do anything.
         continue;
       } else {
         _edges.append(new MoveOperation(in_reg, out_reg));
@@ -314,12 +305,12 @@ class ComputeMoveOrder: public StackObj {
     }
     // Break any cycles in the register moves and emit the in the
     // proper order.
-    compute_store_order(_tmp_vmreg);
+    compute_store_order();
   }
 
   // Walk the edges breaking cycles between moves.  The result list
   // can be walked in order to produce the proper set of loads
-  void compute_store_order(VMStorage temp_register) {
+  void compute_store_order() {
     // Record which moves kill which registers
     KillerTable killer; // a map of VMStorage -> MoveOperation*
     for (int i = 0; i < _edges.length(); i++) {
@@ -328,7 +319,7 @@ class ComputeMoveOrder: public StackObj {
              "multiple moves with the same register as destination");
       killer.put(s->dst(), s);
     }
-    assert(!killer.contains(temp_register),
+    assert(!killer.contains(_tmp_vmreg),
            "make sure temp isn't in the registers that are killed");
 
     // create links between loads and stores
@@ -350,7 +341,7 @@ class ComputeMoveOrder: public StackObj {
           start = start->prev();
         }
         if (start->prev() == s) {
-          start->break_cycle(temp_register);
+          start->break_cycle(_tmp_vmreg);
         }
         // walk the chain forward inserting to store list
         while (start != nullptr) {
@@ -364,31 +355,18 @@ class ComputeMoveOrder: public StackObj {
   }
 
 public:
-  static GrowableArray<Move> compute_move_order(int total_in_args, const VMStorage* in_regs,
+  static GrowableArray<Move> compute_move_order(const GrowableArray<VMStorage>& in_regs,
-                                                int total_out_args, VMStorage* out_regs,
+                                                const GrowableArray<VMStorage>& out_regs,
-                                                const BasicType* in_sig_bt, VMStorage tmp_vmreg) {
+                                                VMStorage tmp_vmreg) {
-    ComputeMoveOrder cmo(total_in_args, in_regs, total_out_args, out_regs, in_sig_bt, tmp_vmreg);
+    ComputeMoveOrder cmo(in_regs, out_regs, tmp_vmreg);
     cmo.compute();
     return cmo._moves;
   }
 };
 
 ArgumentShuffle::ArgumentShuffle(
-    BasicType* in_sig_bt,
+    const GrowableArray<VMStorage>& in_regs,
-    int num_in_args,
+    const GrowableArray<VMStorage>& out_regs,
-    BasicType* out_sig_bt,
-    int num_out_args,
-    const CallingConventionClosure* input_conv,
-    const CallingConventionClosure* output_conv,
     VMStorage shuffle_temp) {
-
+  _moves = ComputeMoveOrder::compute_move_order(in_regs, out_regs, shuffle_temp);
-  VMStorage* in_regs = NEW_RESOURCE_ARRAY(VMStorage, num_in_args);
-  input_conv->calling_convention(in_sig_bt, in_regs, num_in_args);
-
-  VMStorage* out_regs = NEW_RESOURCE_ARRAY(VMStorage, num_out_args);
-  _out_arg_bytes = output_conv->calling_convention(out_sig_bt, out_regs, num_out_args);
-
-  _moves = ComputeMoveOrder::compute_move_order(num_in_args, in_regs,
-                                                num_out_args, out_regs,
-                                                in_sig_bt, shuffle_temp);
 }

src/hotspot/share/prims/foreignGlobals.hpp

@@ -33,9 +33,8 @@
 
 #include CPU_HEADER(foreignGlobals)
 
-// needs to match StubLocations in Java code.
+// Needs to match jdk.internal.foreign.abi.StubLocations in Java code.
-// placeholder locations to be filled in by
+// Placeholder locations to be filled in by the code gen code.
-// the code gen code
 class StubLocations {
 public:
   enum Location : uint32_t {
@@ -56,11 +55,7 @@ public:
   int data_offset(uint32_t loc) const;
 };
 
-class CallingConventionClosure {
+// C++ 'mirror' of jdk.internal.foreign.abi.UpcallLinker.CallRegs
-public:
-  virtual int calling_convention(const BasicType* sig_bt, VMStorage* regs, int num_args) const = 0;
-};
-
 struct CallRegs {
   GrowableArray<VMStorage> _arg_regs;
   GrowableArray<VMStorage> _ret_regs;
@@ -78,25 +73,32 @@ private:
 public:
   static bool is_foreign_linker_supported();
 
+  // Helpers for translating from the Java to C++ representation
   static const ABIDescriptor parse_abi_descriptor(jobject jabi);
   static const CallRegs parse_call_regs(jobject jconv);
   static VMStorage parse_vmstorage(oop storage);
+
+  // Adapter from SharedRuntime::java_calling_convention to a 'single VMStorage per value' form.
+  // Doesn't assign (invalid) storage for T_VOID entries in the signature, which are instead ignored.
+  static int java_calling_convention(const BasicType* signature, int num_args, GrowableArray<VMStorage>& out_regs);
+
+  // Computes the space (in bytes) that is taken up by stack arguments
+  static int compute_out_arg_bytes(const GrowableArray<VMStorage>& out_regs);
+
+  // Replace placeholders (see class StubLocations above) with actual locations in a stub frame
+  static GrowableArray<VMStorage> replace_place_holders(const GrowableArray<VMStorage>& regs, const StubLocations& locs);
+
+  // The receiver method handle for upcalls is injected manually into the argument list by the upcall stub. We need a
+  // filtered list to generate an argument shuffle for the rest of the arguments.
+  static GrowableArray<VMStorage> upcall_filter_receiver_reg(const GrowableArray<VMStorage>& unfiltered_regs);
+
+  // Oop offsets are not passed on to native code.
+  // Filter out the registers of oop offsets to create a list that we can pass to ArgumentShuffle.
+  static GrowableArray<VMStorage> downcall_filter_offset_regs(const GrowableArray<VMStorage>& regs, BasicType* signature,
+                                                              int num_args, bool& has_objects);
 };
 
-class JavaCallingConvention : public CallingConventionClosure {
+// Helper class useful for generating spills and fills of a set of registers.
-public:
-  int calling_convention(const BasicType* sig_bt, VMStorage* regs, int num_args) const override;
-};
-
-class NativeCallingConvention : public CallingConventionClosure {
-  GrowableArray<VMStorage> _input_regs;
-public:
-  NativeCallingConvention(const GrowableArray<VMStorage>& input_regs)
-   : _input_regs(input_regs) {}
-
-  int calling_convention(const BasicType* sig_bt, VMStorage* out_regs, int num_args) const override;
-};
-
 class RegSpiller {
   GrowableArray<VMStorage> _regs;
   int _spill_size_bytes;
@@ -117,30 +119,35 @@ private:
   static void pd_load_reg(MacroAssembler* masm, int offset, VMStorage reg);
 };
 
-struct Move {
+// Class used to compute and generate a shuffle between 2 lists of VMStorages.
-  VMStorage from;
+// The lists must have the same size.
-  VMStorage to;
+// Each VMStorage in the source list (in_regs) is shuffled into the
-};
+// the VMStorage at the same index in the destination list (out_regs).
-
+// This class helps to automatically compute an order of moves that makes
+// sure not to destroy values accidentally by interfering moves, in case the
+// source and destination registers overlap.
 class ArgumentShuffle {
 private:
+  class ComputeMoveOrder;
+  struct Move {
+    VMStorage from;
+    VMStorage to;
+  };
+
   GrowableArray<Move> _moves;
-  int _out_arg_bytes;
 public:
   ArgumentShuffle(
-    BasicType* in_sig_bt, int num_in_args,
+    const GrowableArray<VMStorage>& in_regs,
-    BasicType* out_sig_bt, int num_out_args,
+    const GrowableArray<VMStorage>& out_regs,
-    const CallingConventionClosure* input_conv, const CallingConventionClosure* output_conv,
     VMStorage shuffle_temp);
 
-  int out_arg_bytes() const { return _out_arg_bytes; }
+  void generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const {
-  void generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const {
+    pd_generate(masm, tmp, in_stk_bias, out_stk_bias);
-    pd_generate(masm, tmp, in_stk_bias, out_stk_bias, locs);
   }
 
   void print_on(outputStream* os) const;
 private:
-  void pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const;
+  void pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const;
 };
 
 #endif // SHARE_PRIMS_FOREIGN_GLOBALS

src/hotspot/share/prims/nativeEntryPoint.cpp

@@ -52,10 +52,12 @@ JNI_ENTRY(jlong, NEP_makeDowncallStub(JNIEnv* env, jclass _unused, jobject metho
   GrowableArray<VMStorage> input_regs(pcount);
   for (int i = 0, bt_idx = 0; i < pcount; i++) {
     oop type_oop = java_lang_invoke_MethodType::ptype(type, i);
-    assert(java_lang_Class::is_primitive(type_oop), "Only primitives expected");
+    BasicType bt = java_lang_Class::as_BasicType(type_oop);
-    BasicType bt = java_lang_Class::primitive_type(type_oop);
     basic_type[bt_idx++] = bt;
-    input_regs.push(ForeignGlobals::parse_vmstorage(arg_moves_oop->obj_at(i)));
+    oop reg_oop = arg_moves_oop->obj_at(i);
+    if (reg_oop != nullptr) {
+      input_regs.push(ForeignGlobals::parse_vmstorage(reg_oop));
+    }
 
     if (bt == BasicType::T_DOUBLE || bt == BasicType::T_LONG) {
       basic_type[bt_idx++] = T_VOID;

src/hotspot/share/prims/upcallLinker.cpp

@@ -178,12 +178,10 @@ JVM_ENTRY(jlong, UL_MakeUpcallStub(JNIEnv *env, jclass unused, jobject mh, jobje
     assert(i == total_out_args, "");
     ret_type = ss.type();
   }
-  // skip receiver
-  BasicType* in_sig_bt = out_sig_bt + 1;
-  int total_in_args = total_out_args - 1;
 
   return (jlong) UpcallLinker::make_upcall_stub(
-    mh_j, entry, in_sig_bt, total_in_args, out_sig_bt, total_out_args, ret_type, abi, conv, needs_return_buffer, checked_cast<int>(ret_buf_size));
+    mh_j, entry, out_sig_bt, total_out_args, ret_type,
+    abi, conv, needs_return_buffer, checked_cast<int>(ret_buf_size));
 JVM_END
 
 #define CC (char*)  /*cast a literal from (const char*)*/

src/hotspot/share/prims/upcallLinker.hpp

@@ -38,7 +38,6 @@ private:
   static void on_exit(UpcallStub::FrameData* context);
 public:
   static address make_upcall_stub(jobject mh, Method* entry,
-                                  BasicType* in_sig_bt, int total_in_args,
                                   BasicType* out_sig_bt, int total_out_args,
                                   BasicType ret_type,
                                   jobject jabi, jobject jconv,

src/java.base/share/classes/java/lang/foreign/Linker.java

@@ -669,8 +669,13 @@ public sealed interface Linker permits AbstractLinker {
      * might attempt to access the contents of the segment. As such, one of the
      * exceptions specified by the {@link MemorySegment#get(ValueLayout.OfByte, long)} or
      * the {@link MemorySegment#copy(MemorySegment, long, MemorySegment, long, long)}
-     * methods may be thrown. The returned method handle will additionally throw
+     * methods may be thrown. If an argument is a {@link MemorySegment} whose
-     * {@link NullPointerException} if any argument passed to it is {@code null}.
+     * corresponding layout is an {@linkplain AddressLayout address layout}, the linker
+     * will throw an {@link IllegalArgumentException} if the segment is a heap memory
+     * segment, unless heap memory segments are explicitly allowed through the
+     * {@link Linker.Option#critical(boolean)} linker option. The returned method handle
+     * will additionally throw {@link NullPointerException} if any argument passed to it
+     * is {@code null}.
      *
      * @param function the function descriptor of the target foreign function
      * @param options  the linker options associated with this linkage request
@@ -914,9 +919,23 @@ public sealed interface Linker permits AbstractLinker {
          * <p>
          * Using this linker option when linking non-critical functions is likely to have
          * adverse effects, such as loss of performance or JVM crashes.
+         * <p>
+         * Critical functions can optionally allow access to the Java heap. This allows
+         * clients to pass heap memory segments as addresses, where normally only off-heap
+         * memory segments would be allowed. The memory region inside the Java heap is
+         * exposed through a temporary native address that is valid for the duration of
+         * the function call. Use of this mechanism is therefore only recommended when a
+         * function needs to do short-lived access to Java heap memory, and copying the
+         * relevant data to an off-heap memory segment would be prohibitive in terms of
+         * performance.
+         *
+         * @param allowHeapAccess whether the linked function should allow access to the
+         *                        Java heap.
          */
-        static Option critical() {
+        static Option critical(boolean allowHeapAccess) {
-            return LinkerOptions.Critical.INSTANCE;
+            return allowHeapAccess
+                ? LinkerOptions.Critical.ALLOW_HEAP
+                : LinkerOptions.Critical.DONT_ALLOW_HEAP;
         }
     }
 }

src/java.base/share/classes/java/lang/invoke/NativeMethodHandle.java

@@ -77,7 +77,8 @@ import static java.lang.invoke.MethodHandleStatics.newInternalError;
               || pType == int.class
               || pType == float.class
               || pType == double.class
-              || pType == void.class);
+              || pType == void.class
+              || pType == Object.class);
     }
 
     private static final MemberName.Factory IMPL_NAMES = MemberName.getFactory();

src/java.base/share/classes/jdk/internal/foreign/abi/Binding.java

@@ -24,6 +24,7 @@
  */
 package jdk.internal.foreign.abi;
 
+import jdk.internal.foreign.AbstractMemorySegmentImpl;
 import jdk.internal.foreign.Utils;
 import jdk.internal.foreign.abi.BindingInterpreter.LoadFunc;
 import jdk.internal.foreign.abi.BindingInterpreter.StoreFunc;
@@ -201,7 +202,7 @@ public sealed interface Binding {
                    LoadFunc loadFunc, SegmentAllocator allocator);
 
     private static void checkType(Class<?> type) {
-        if (!type.isPrimitive() || type == void.class)
+        if (type != Object.class && (!type.isPrimitive() || type == void.class))
             throw new IllegalArgumentException("Illegal type: " + type);
     }
 
@@ -267,8 +268,21 @@ public sealed interface Binding {
         return new BoxAddress(byteSize, 1, true);
     }
 
-    static UnboxAddress unboxAddress() {
+    // alias
-        return UnboxAddress.INSTANCE;
+    static SegmentOffset unboxAddress() {
+        return segmentOffsetNoAllowHeap();
+    }
+
+    static SegmentBase segmentBase() {
+        return SegmentBase.INSTANCE;
+    }
+
+    static SegmentOffset segmentOffsetAllowHeap() {
+        return SegmentOffset.INSTANCE_ALLOW_HEAP;
+    }
+
+    static SegmentOffset segmentOffsetNoAllowHeap() {
+        return SegmentOffset.INSTANCE_NO_ALLOW_HEAP;
     }
 
     static Dup dup() {
@@ -413,6 +427,21 @@ public sealed interface Binding {
             return this;
         }
 
+        public Binding.Builder segmentBase() {
+            bindings.add(Binding.segmentBase());
+            return this;
+        }
+
+        public Binding.Builder segmentOffsetAllowHeap() {
+            bindings.add(Binding.segmentOffsetAllowHeap());
+            return this;
+        }
+
+        public Binding.Builder segmentOffsetNoAllowHeap() {
+            bindings.add(Binding.segmentOffsetNoAllowHeap());
+            return this;
+        }
+
         public Binding.Builder dup() {
             bindings.add(Binding.dup());
             return this;
@@ -448,8 +477,10 @@ public sealed interface Binding {
 
     /**
      * VM_STORE([storage location], [type])
-     * Pops a [type] from the operand stack, and moves it to [storage location]
+     *   Pops a [type] from the operand stack, and moves it to [storage location]
-     * The [type] must be one of byte, short, char, int, long, float, or double
+     *   The [type] must be one of byte, short, char, int, long, float, or double.
+     *   [storage location] may be 'null', indicating that this value should be passed
+     *   to the VM stub, but does not have an explicit target register (e.g. oop offsets)
      */
     record VMStore(VMStorage storage, Class<?> type) implements Move {
 
@@ -469,8 +500,8 @@ public sealed interface Binding {
 
     /**
      * VM_LOAD([storage location], [type])
-     * Loads a [type] from [storage location], and pushes it onto the operand stack.
+     *   Loads a [type] from [storage location], and pushes it onto the operand stack.
-     * The [type] must be one of byte, short, char, int, long, float, or double
+     *   The [type] must be one of byte, short, char, int, long, float, or double
      */
     record VMLoad(VMStorage storage, Class<?> type) implements Move {
 
@@ -493,9 +524,9 @@ public sealed interface Binding {
 
     /**
      * BUFFER_STORE([offset into memory region], [type], [width])
-     * Pops a [type] from the operand stack, then pops a MemorySegment from the operand stack.
+     *   Pops a [type] from the operand stack, then pops a MemorySegment from the operand stack.
-     * Stores [width] bytes of the value contained in the [type] to [offset into memory region].
+     *   Stores [width] bytes of the value contained in the [type] to [offset into memory region].
-     * The [type] must be one of byte, short, char, int, long, float, or double
+     *   The [type] must be one of byte, short, char, int, long, float, or double
      */
     record BufferStore(long offset, Class<?> type, int byteWidth) implements Dereference {
 
@@ -550,9 +581,9 @@ public sealed interface Binding {
 
     /**
      * BUFFER_LOAD([offset into memory region], [type], [width])
-     * Pops a MemorySegment from the operand stack,
+     *   Pops a MemorySegment from the operand stack,
-     * and then loads [width] bytes from it at [offset into memory region], into a [type].
+     *   and then loads [width] bytes from it at [offset into memory region], into a [type].
-     * The [type] must be one of byte, short, char, int, long, float, or double
+     *   The [type] must be one of byte, short, char, int, long, float, or double
      */
     record BufferLoad(long offset, Class<?> type, int byteWidth) implements Dereference {
 
@@ -606,8 +637,8 @@ public sealed interface Binding {
     /**
      * COPY([size], [alignment])
      *   Creates a new MemorySegment with the given [size] and [alignment],
-     *     and copies contents from a MemorySegment popped from the top of the operand stack into this new buffer,
+     *   and copies contents from a MemorySegment popped from the top of the operand stack into this new buffer,
-     *     and pushes the new buffer onto the operand stack
+     *   and pushes the new buffer onto the operand stack
      */
     record Copy(long size, long alignment) implements Binding {
         private static MemorySegment copyBuffer(MemorySegment operand, long size, long alignment, SegmentAllocator allocator) {
@@ -653,12 +684,37 @@ public sealed interface Binding {
     }
 
     /**
-     * UNBOX_ADDRESS()
+     * SEGMENT_BASE()
-     * Pops a 'MemoryAddress' from the operand stack, converts it to a 'long',
+     *   Pops a MemorySegment from the stack, retrieves the heap base object from it, or null if there is none
-     * with the given size, and pushes that onto the operand stack
+     *   (See: AbstractMemorySegmentImpl::unsafeGetBase), and pushes the result onto the operand stack.
      */
-    record UnboxAddress() implements Binding {
+    record SegmentBase() implements Binding {
-        static final UnboxAddress INSTANCE = new UnboxAddress();
+        static final SegmentBase INSTANCE = new SegmentBase();
+
+        @Override
+        public void verify(Deque<Class<?>> stack) {
+            Class<?> actualType = stack.pop();
+            SharedUtils.checkType(actualType, MemorySegment.class);
+            stack.push(Object.class);
+        }
+
+        @Override
+        public void interpret(Deque<Object> stack, StoreFunc storeFunc,
+                              LoadFunc loadFunc, SegmentAllocator allocator) {
+            stack.push(((AbstractMemorySegmentImpl)stack.pop()).unsafeGetBase());
+        }
+    }
+
+    /**
+     * SEGMENT_OFFSET([allowHeap])
+     *   Pops a MemorySegment from the stack, retrieves the offset from it,
+     *   (See: AbstractMemorySegmentImpl::unsafeGetOffset), and pushes the result onto the operand stack.
+     *   Note that for heap segments, the offset is a virtual address into the heap base object.
+     *   If [allowHeap] is 'false' an exception will be thrown for heap segments (See SharedUtils::checkNative).
+     */
+    record SegmentOffset(boolean allowHeap) implements Binding {
+        static final SegmentOffset INSTANCE_NO_ALLOW_HEAP = new SegmentOffset(false);
+        static final SegmentOffset INSTANCE_ALLOW_HEAP = new SegmentOffset(true);
 
         @Override
         public void verify(Deque<Class<?>> stack) {
@@ -670,14 +726,18 @@ public sealed interface Binding {
         @Override
         public void interpret(Deque<Object> stack, StoreFunc storeFunc,
                               LoadFunc loadFunc, SegmentAllocator allocator) {
-            stack.push(SharedUtils.unboxSegment((MemorySegment)stack.pop()));
+            MemorySegment operand = (MemorySegment) stack.pop();
+            if (!allowHeap) {
+                SharedUtils.checkNative(operand);
+            }
+            stack.push(((AbstractMemorySegmentImpl)operand).unsafeGetOffset());
         }
     }
 
     /**
      * BOX_ADDRESS()
-     * Pops a 'long' from the operand stack, converts it to a 'MemorySegment', with the given size and memory scope
+     *   Pops a 'long' from the operand stack, converts it to a 'MemorySegment', with the given size and memory scope
-     * (either the context scope, or the global scope), and pushes that onto the operand stack.
+     *   (either the context scope, or the global scope), and pushes that onto the operand stack.
      */
     record BoxAddress(long size, long align, boolean needsScope) implements Binding {
 

src/java.base/share/classes/jdk/internal/foreign/abi/BindingInterpreter.java

@@ -27,12 +27,13 @@ package jdk.internal.foreign.abi;
 import java.lang.foreign.SegmentAllocator;
 import java.util.ArrayDeque;
 import java.util.Deque;
+import java.util.LinkedList;
 import java.util.List;
 
 public class BindingInterpreter {
 
     static void unbox(Object arg, List<Binding> bindings, StoreFunc storeFunc, SegmentAllocator allocator) {
-        Deque<Object> stack = new ArrayDeque<>();
+        Deque<Object> stack = new LinkedList<>(); // Use LinkedList as a null-friendly Deque for null segment bases
 
         stack.push(arg);
         for (Binding b : bindings) {

src/java.base/share/classes/jdk/internal/foreign/abi/BindingSpecializer.java

@@ -39,9 +39,10 @@ import jdk.internal.foreign.abi.Binding.BufferStore;
 import jdk.internal.foreign.abi.Binding.Cast;
 import jdk.internal.foreign.abi.Binding.Copy;
 import jdk.internal.foreign.abi.Binding.Dup;
+import jdk.internal.foreign.abi.Binding.SegmentBase;
+import jdk.internal.foreign.abi.Binding.SegmentOffset;
 import jdk.internal.foreign.abi.Binding.ShiftLeft;
 import jdk.internal.foreign.abi.Binding.ShiftRight;
-import jdk.internal.foreign.abi.Binding.UnboxAddress;
 import jdk.internal.foreign.abi.Binding.VMLoad;
 import jdk.internal.foreign.abi.Binding.VMStore;
 import sun.security.action.GetBooleanAction;
@@ -103,7 +104,9 @@ public class BindingSpecializer {
     private static final MethodTypeDesc MTD_SCOPE = MethodTypeDesc.of(CD_MemorySegment_Scope);
     private static final MethodTypeDesc MTD_SESSION_IMPL = MethodTypeDesc.of(CD_MemorySessionImpl);
     private static final MethodTypeDesc MTD_CLOSE = MTD_void;
-    private static final MethodTypeDesc MTD_UNBOX_SEGMENT = MethodTypeDesc.of(CD_long, CD_MemorySegment);
+    private static final MethodTypeDesc MTD_CHECK_NATIVE = MethodTypeDesc.of(CD_void, CD_MemorySegment);
+    private static final MethodTypeDesc MTD_UNSAFE_GET_BASE = MethodTypeDesc.of(CD_Object);
+    private static final MethodTypeDesc MTD_UNSAFE_GET_OFFSET = MethodTypeDesc.of(CD_long);
     private static final MethodTypeDesc MTD_COPY = MethodTypeDesc.of(CD_void, CD_MemorySegment, CD_long, CD_MemorySegment, CD_long, CD_long);
     private static final MethodTypeDesc MTD_LONG_TO_ADDRESS_NO_SCOPE = MethodTypeDesc.of(CD_MemorySegment, CD_long, CD_long, CD_long);
     private static final MethodTypeDesc MTD_LONG_TO_ADDRESS_SCOPE = MethodTypeDesc.of(CD_MemorySegment, CD_long, CD_long, CD_long, CD_MemorySessionImpl);
@@ -456,18 +459,19 @@ public class BindingSpecializer {
     private void doBindings(List<Binding> bindings) {
         for (Binding binding : bindings) {
             switch (binding) {
-                case VMStore vmStore         -> emitVMStore(vmStore);
+                case VMStore vmStore             -> emitVMStore(vmStore);
-                case VMLoad vmLoad           -> emitVMLoad(vmLoad);
+                case VMLoad vmLoad               -> emitVMLoad(vmLoad);
-                case BufferStore bufferStore -> emitBufferStore(bufferStore);
+                case BufferStore bufferStore     -> emitBufferStore(bufferStore);
-                case BufferLoad bufferLoad   -> emitBufferLoad(bufferLoad);
+                case BufferLoad bufferLoad       -> emitBufferLoad(bufferLoad);
-                case Copy copy               -> emitCopyBuffer(copy);
+                case Copy copy                   -> emitCopyBuffer(copy);
-                case Allocate allocate       -> emitAllocBuffer(allocate);
+                case Allocate allocate           -> emitAllocBuffer(allocate);
-                case BoxAddress boxAddress   -> emitBoxAddress(boxAddress);
+                case BoxAddress boxAddress       -> emitBoxAddress(boxAddress);
-                case UnboxAddress unused     -> emitUnboxAddress();
+                case SegmentBase unused          -> emitSegmentBase();
-                case Dup unused              -> emitDupBinding();
+                case SegmentOffset segmentOffset -> emitSegmentOffset(segmentOffset);
-                case ShiftLeft shiftLeft     -> emitShiftLeft(shiftLeft);
+                case Dup unused                  -> emitDupBinding();
-                case ShiftRight shiftRight   -> emitShiftRight(shiftRight);
+                case ShiftLeft shiftLeft         -> emitShiftLeft(shiftLeft);
-                case Cast cast               -> emitCast(cast);
+                case ShiftRight shiftRight       -> emitShiftRight(shiftRight);
+                case Cast cast                   -> emitCast(cast);
             }
         }
     }
@@ -775,9 +779,23 @@ public class BindingSpecializer {
         pushType(toType);
     }
 
-    private void emitUnboxAddress() {
+    private void emitSegmentBase() {
         popType(MemorySegment.class);
-        cb.invokestatic(CD_SharedUtils, "unboxSegment", MTD_UNBOX_SEGMENT);
+        cb.checkcast(CD_AbstractMemorySegmentImpl);
+        cb.invokevirtual(CD_AbstractMemorySegmentImpl, "unsafeGetBase", MTD_UNSAFE_GET_BASE);
+        pushType(Object.class);
+    }
+
+    private void emitSegmentOffset(SegmentOffset segmentOffset) {
+        popType(MemorySegment.class);
+
+        if (!segmentOffset.allowHeap()) {
+            cb.dup();
+            cb.invokestatic(CD_SharedUtils, "checkNative", MTD_CHECK_NATIVE);
+        }
+        cb.checkcast(CD_AbstractMemorySegmentImpl);
+        cb.invokevirtual(CD_AbstractMemorySegmentImpl, "unsafeGetOffset", MTD_UNSAFE_GET_OFFSET);
+
         pushType(long.class);
     }
 

src/java.base/share/classes/jdk/internal/foreign/abi/CallingSequenceBuilder.java

@@ -25,16 +25,17 @@
 package jdk.internal.foreign.abi;
 
 import jdk.internal.foreign.Utils;
-import jdk.internal.foreign.abi.Binding.Allocate;
+import jdk.internal.foreign.abi.Binding.*;
 import jdk.internal.foreign.abi.Binding.BoxAddress;
 import jdk.internal.foreign.abi.Binding.BufferLoad;
 import jdk.internal.foreign.abi.Binding.BufferStore;
 import jdk.internal.foreign.abi.Binding.Cast;
 import jdk.internal.foreign.abi.Binding.Copy;
 import jdk.internal.foreign.abi.Binding.Dup;
+import jdk.internal.foreign.abi.Binding.SegmentBase;
+import jdk.internal.foreign.abi.Binding.SegmentOffset;
 import jdk.internal.foreign.abi.Binding.ShiftLeft;
 import jdk.internal.foreign.abi.Binding.ShiftRight;
-import jdk.internal.foreign.abi.Binding.UnboxAddress;
 import jdk.internal.foreign.abi.Binding.VMLoad;
 import jdk.internal.foreign.abi.Binding.VMStore;
 import sun.security.action.GetPropertyAction;
@@ -217,19 +218,19 @@ public class CallingSequenceBuilder {
 
     static boolean isUnbox(Binding binding) {
         return switch (binding) {
-            case VMStore      unused -> true;
+            case VMStore       unused -> true;
-            case BufferLoad   unused -> true;
+            case BufferLoad    unused -> true;
-            case Copy         unused -> true;
+            case Copy          unused -> true;
-            case UnboxAddress unused -> true;
+            case Dup           unused -> true;
-            case Dup          unused -> true;
+            case SegmentBase   unused -> true;
-            case ShiftLeft    unused -> true;
+            case SegmentOffset unused -> true;
-            case ShiftRight   unused -> true;
+            case ShiftLeft     unused -> true;
-            case Cast         unused -> true;
+            case ShiftRight    unused -> true;
-
+            case Cast          unused -> true;
-            case VMLoad       unused -> false;
+            case VMLoad        unused -> false;
-            case BufferStore  unused -> false;
+            case BufferStore   unused -> false;
-            case Allocate     unused -> false;
+            case Allocate      unused -> false;
-            case BoxAddress   unused -> false;
+            case BoxAddress    unused -> false;
         };
     }
 
@@ -252,19 +253,20 @@ public class CallingSequenceBuilder {
 
     static boolean isBox(Binding binding) {
         return switch (binding) {
-            case VMLoad       unused -> true;
+            case VMLoad        unused -> true;
-            case BufferStore  unused -> true;
+            case BufferStore   unused -> true;
-            case Copy         unused -> true;
+            case Copy          unused -> true;
-            case Allocate     unused -> true;
+            case Allocate      unused -> true;
-            case BoxAddress   unused -> true;
+            case BoxAddress    unused -> true;
-            case Dup          unused -> true;
+            case Dup           unused -> true;
-            case ShiftLeft    unused -> true;
+            case ShiftLeft     unused -> true;
-            case ShiftRight   unused -> true;
+            case ShiftRight    unused -> true;
-            case Cast         unused -> true;
+            case Cast          unused -> true;
 
-            case VMStore      unused -> false;
+            case VMStore       unused -> false;
-            case BufferLoad   unused -> false;
+            case BufferLoad    unused -> false;
-            case UnboxAddress unused -> false;
+            case SegmentBase   unused -> false;
+            case SegmentOffset unused -> false;
         };
     }
 

src/java.base/share/classes/jdk/internal/foreign/abi/DowncallLinker.java

@@ -98,9 +98,7 @@ public class DowncallLinker {
         if (USE_SPEC) {
             handle = BindingSpecializer.specializeDowncall(handle, callingSequence, abi);
          } else {
-            Map<VMStorage, Integer> argIndexMap = SharedUtils.indexMap(argMoves);
+            InvocationData invData = new InvocationData(handle, callingSequence);
-
-            InvocationData invData = new InvocationData(handle, callingSequence, argIndexMap);
             handle = insertArguments(MH_INVOKE_INTERP_BINDINGS.bindTo(this), 2, invData);
             MethodType interpType = callingSequence.callerMethodType();
             if (callingSequence.needsReturnBuffer()) {
@@ -151,7 +149,7 @@ public class DowncallLinker {
         return Arrays.stream(moves).map(Binding.Move::storage).toArray(VMStorage[]::new);
     }
 
-    private record InvocationData(MethodHandle leaf, CallingSequence callingSequence, Map<VMStorage, Integer> argIndexMap) {}
+    private record InvocationData(MethodHandle leaf, CallingSequence callingSequence) {}
 
     Object invokeInterpBindings(SegmentAllocator allocator, Object[] args, InvocationData invData) throws Throwable {
         Arena unboxArena = callingSequence.allocationSize() != 0
@@ -172,6 +170,13 @@ public class DowncallLinker {
             }
 
             Object[] leafArgs = new Object[invData.leaf.type().parameterCount()];
+            BindingInterpreter.StoreFunc storeFunc = new BindingInterpreter.StoreFunc() {
+                    int argOffset = 0;
+                    @Override
+                    public void store(VMStorage storage, Object o) {
+                        leafArgs[argOffset++] = o;
+                    }
+                };
             for (int i = 0; i < args.length; i++) {
                 Object arg = args[i];
                 if (callingSequence.functionDesc().argumentLayouts().get(i) instanceof AddressLayout) {
@@ -183,8 +188,7 @@ public class DowncallLinker {
                         acquiredScopes.add(sessionImpl);
                     }
                 }
-                BindingInterpreter.unbox(arg, callingSequence.argumentBindings(i),
+                BindingInterpreter.unbox(arg, callingSequence.argumentBindings(i), storeFunc, unboxArena);
-                    (storage, value) -> leafArgs[invData.argIndexMap.get(storage)] = value, unboxArena);
             }
 
             // call leaf

src/java.base/share/classes/jdk/internal/foreign/abi/LinkerOptions.java

@@ -106,6 +106,11 @@ public class LinkerOptions {
         return c != null;
     }
 
+    public boolean allowsHeapAccess() {
+        Critical c = getOption(Critical.class);
+        return c != null && c.allowHeapAccess();
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -145,8 +150,9 @@ public class LinkerOptions {
         }
     }
 
-    public record Critical() implements LinkerOptionImpl {
+    public record Critical(boolean allowHeapAccess) implements LinkerOptionImpl {
-        public static Critical INSTANCE = new Critical();
+        public static Critical ALLOW_HEAP = new Critical(true);
+        public static Critical DONT_ALLOW_HEAP = new Critical(false);
 
         @Override
         public void validateForDowncall(FunctionDescriptor descriptor) {

src/java.base/share/classes/jdk/internal/foreign/abi/SharedUtils.java

@@ -314,10 +314,14 @@ public final class SharedUtils {
         }
     }
 
-    public static long unboxSegment(MemorySegment segment) {
+    public static void checkNative(MemorySegment segment) {
         if (!segment.isNative()) {
             throw new IllegalArgumentException("Heap segment not allowed: " + segment);
         }
+    }
+
+    public static long unboxSegment(MemorySegment segment) {
+        checkNative(segment);
         return segment.address();
     }
 

src/java.base/share/classes/jdk/internal/foreign/abi/aarch64/CallArranger.java

@@ -151,8 +151,8 @@ public abstract class CallArranger {
 
         boolean forVariadicFunction = options.isVariadicFunction();
 
-        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, forVariadicFunction);
+        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, forVariadicFunction, options.allowsHeapAccess());
-        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, forVariadicFunction) : new BoxBindingCalculator(false);
+        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, forVariadicFunction, false) : new BoxBindingCalculator(false);
 
         boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout());
         if (returnInMemory) {
@@ -388,11 +388,13 @@ public abstract class CallArranger {
     class UnboxBindingCalculator extends BindingCalculator {
         protected final boolean forArguments;
         protected final boolean forVariadicFunction;
+        private final boolean useAddressPairs;
 
-        UnboxBindingCalculator(boolean forArguments, boolean forVariadicFunction) {
+        UnboxBindingCalculator(boolean forArguments, boolean forVariadicFunction, boolean useAddressPairs) {
             super(forArguments, forVariadicFunction);
             this.forArguments = forArguments;
             this.forVariadicFunction = forVariadicFunction;
+            this.useAddressPairs = useAddressPairs;
         }
 
         @Override
@@ -432,9 +434,17 @@ public abstract class CallArranger {
                     bindings.vmStore(storage, long.class);
                 }
                 case POINTER -> {
-                    bindings.unboxAddress();
                     VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER, (ValueLayout) layout);
-                    bindings.vmStore(storage, long.class);
+                    if (useAddressPairs) {
+                        bindings.dup()
+                                .segmentBase()
+                                .vmStore(storage, Object.class)
+                                .segmentOffsetAllowHeap()
+                                .vmStore(null, long.class);
+                    } else {
+                        bindings.unboxAddress();
+                        bindings.vmStore(storage, long.class);
+                    }
                 }
                 case INTEGER -> {
                     VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER, (ValueLayout) layout);

src/java.base/share/classes/jdk/internal/foreign/abi/fallback/FallbackLinker.java

@@ -24,13 +24,6 @@
  */
 package jdk.internal.foreign.abi.fallback;
 
-import jdk.internal.foreign.AbstractMemorySegmentImpl;
-import jdk.internal.foreign.MemorySessionImpl;
-import jdk.internal.foreign.abi.AbstractLinker;
-import jdk.internal.foreign.abi.CapturableState;
-import jdk.internal.foreign.abi.LinkerOptions;
-import jdk.internal.foreign.abi.SharedUtils;
-
 import java.lang.foreign.AddressLayout;
 import java.lang.foreign.Arena;
 import java.lang.foreign.FunctionDescriptor;
@@ -39,16 +32,6 @@ import java.lang.foreign.MemoryLayout;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.SegmentAllocator;
 import java.lang.foreign.ValueLayout;
-import java.lang.invoke.MethodHandle;
-import java.lang.invoke.MethodHandles;
-import java.lang.invoke.MethodType;
-import java.lang.ref.Reference;
-import java.nio.ByteOrder;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Consumer;
-
 import static java.lang.foreign.ValueLayout.ADDRESS;
 import static java.lang.foreign.ValueLayout.JAVA_BOOLEAN;
 import static java.lang.foreign.ValueLayout.JAVA_BYTE;
@@ -58,7 +41,21 @@ import static java.lang.foreign.ValueLayout.JAVA_FLOAT;
 import static java.lang.foreign.ValueLayout.JAVA_INT;
 import static java.lang.foreign.ValueLayout.JAVA_LONG;
 import static java.lang.foreign.ValueLayout.JAVA_SHORT;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
 import static java.lang.invoke.MethodHandles.foldArguments;
+import java.lang.invoke.MethodType;
+import java.lang.ref.Reference;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import jdk.internal.foreign.AbstractMemorySegmentImpl;
+import jdk.internal.foreign.MemorySessionImpl;
+import jdk.internal.foreign.abi.AbstractLinker;
+import jdk.internal.foreign.abi.CapturableState;
+import jdk.internal.foreign.abi.LinkerOptions;
+import jdk.internal.foreign.abi.SharedUtils;
 
 public final class FallbackLinker extends AbstractLinker {
 
@@ -95,7 +92,7 @@ public final class FallbackLinker extends AbstractLinker {
                 .mapToInt(CapturableState::mask)
                 .reduce(0, (a, b) -> a | b);
         DowncallData invData = new DowncallData(cif, function.returnLayout().orElse(null),
-                function.argumentLayouts(), capturedStateMask);
+                function.argumentLayouts(), capturedStateMask, options.allowsHeapAccess());
 
         MethodHandle target = MethodHandles.insertArguments(MH_DO_DOWNCALL, 2, invData);
 
@@ -155,12 +152,13 @@ public final class FallbackLinker extends AbstractLinker {
     }
 
     private record DowncallData(MemorySegment cif, MemoryLayout returnLayout, List<MemoryLayout> argLayouts,
-                                int capturedStateMask) {}
+                                int capturedStateMask, boolean allowsHeapAccess) {}
 
     private static Object doDowncall(SegmentAllocator returnAllocator, Object[] args, DowncallData invData) {
         List<MemorySessionImpl> acquiredSessions = new ArrayList<>();
         try (Arena arena = Arena.ofConfined()) {
             int argStart = 0;
+            Object[] heapBases = invData.allowsHeapAccess() ? new Object[args.length] : null;
 
             MemorySegment target = (MemorySegment) args[argStart++];
             MemorySessionImpl targetImpl = ((AbstractMemorySegmentImpl) target).sessionImpl();
@@ -180,12 +178,22 @@ public final class FallbackLinker extends AbstractLinker {
             for (int i = 0; i < argLayouts.size(); i++) {
                 Object arg = args[argStart + i];
                 MemoryLayout layout = argLayouts.get(i);
-                MemorySegment argSeg = arena.allocate(layout);
+
-                writeValue(arg, layout, argSeg, addr -> {
+                if (layout instanceof AddressLayout) {
-                    MemorySessionImpl sessionImpl = ((AbstractMemorySegmentImpl) addr).sessionImpl();
+                    AbstractMemorySegmentImpl ms = (AbstractMemorySegmentImpl) arg;
+                    MemorySessionImpl sessionImpl = ms.sessionImpl();
                     sessionImpl.acquire0();
                     acquiredSessions.add(sessionImpl);
-                });
+                    if (invData.allowsHeapAccess() && !ms.isNative()) {
+                        heapBases[i] = ms.unsafeGetBase();
+                        // write the offset to the arg segment, add array ptr to it in native code
+                        layout = JAVA_LONG;
+                        arg = ms.address();
+                    }
+                }
+
+                MemorySegment argSeg = arena.allocate(layout);
+                writeValue(arg, layout, argSeg);
                 argPtrs.setAtIndex(ADDRESS, i, argSeg);
             }
 
@@ -194,7 +202,8 @@ public final class FallbackLinker extends AbstractLinker {
                 retSeg = (invData.returnLayout() instanceof GroupLayout ? returnAllocator : arena).allocate(invData.returnLayout);
             }
 
-            LibFallback.doDowncall(invData.cif, target, retSeg, argPtrs, capturedState, invData.capturedStateMask());
+            LibFallback.doDowncall(invData.cif, target, retSeg, argPtrs, capturedState, invData.capturedStateMask(),
+                                   heapBases, args.length);
 
             Reference.reachabilityFence(invData.cif());
 
@@ -235,11 +244,6 @@ public final class FallbackLinker extends AbstractLinker {
 
     // where
     private static void writeValue(Object arg, MemoryLayout layout, MemorySegment argSeg) {
-        writeValue(arg, layout, argSeg, addr -> {});
-    }
-
-    private static void writeValue(Object arg, MemoryLayout layout, MemorySegment argSeg,
-                                   Consumer<MemorySegment> acquireCallback) {
         switch (layout) {
             case ValueLayout.OfBoolean bl -> argSeg.set(bl, 0, (Boolean) arg);
             case ValueLayout.OfByte    bl -> argSeg.set(bl, 0, (Byte) arg);
@@ -249,11 +253,7 @@ public final class FallbackLinker extends AbstractLinker {
             case ValueLayout.OfLong    ll -> argSeg.set(ll, 0, (Long) arg);
             case ValueLayout.OfFloat   fl -> argSeg.set(fl, 0, (Float) arg);
             case ValueLayout.OfDouble  dl -> argSeg.set(dl, 0, (Double) arg);
-            case AddressLayout         al -> {
+            case AddressLayout         al -> argSeg.set(al, 0, (MemorySegment) arg);
-                MemorySegment addrArg = (MemorySegment) arg;
-                acquireCallback.accept(addrArg);
-                argSeg.set(al, 0, addrArg);
-            }
             case GroupLayout            _ ->
                     MemorySegment.copy((MemorySegment) arg, 0, argSeg, 0, argSeg.byteSize()); // by-value struct
             case null, default -> {

src/java.base/share/classes/jdk/internal/foreign/abi/fallback/LibFallback.java

@@ -93,10 +93,12 @@ final class LibFallback {
      * @see jdk.internal.foreign.abi.CapturableState
      */
     static void doDowncall(MemorySegment cif, MemorySegment target, MemorySegment retPtr, MemorySegment argPtrs,
-                                  MemorySegment capturedState, int capturedStateMask) {
+                           MemorySegment capturedState, int capturedStateMask,
+                           Object[] heapBases, int numArgs) {
             doDowncall(cif.address(), target.address(),
-                    retPtr == null ? 0 : retPtr.address(), argPtrs.address(),
+                       retPtr == null ? 0 : retPtr.address(), argPtrs.address(),
-                    capturedState == null ? 0 : capturedState.address(), capturedStateMask);
+                       capturedState == null ? 0 : capturedState.address(), capturedStateMask,
+                       heapBases, numArgs);
     }
 
     /**
@@ -210,7 +212,9 @@ final class LibFallback {
 
     private static native int createClosure(long cif, Object userData, long[] ptrs);
     private static native void freeClosure(long closureAddress, long globalTarget);
-    private static native void doDowncall(long cif, long fn, long rvalue, long avalues, long capturedState, int capturedStateMask);
+    private static native void doDowncall(long cif, long fn, long rvalue, long avalues,
+                                          long capturedState, int capturedStateMask,
+                                          Object[] heapBases, int numArgs);
 
     private static native int ffi_prep_cif(long cif, int abi, int nargs, long rtype, long atypes);
     private static native int ffi_prep_cif_var(long cif, int abi, int nfixedargs, int ntotalargs, long rtype, long atypes);

src/java.base/share/classes/jdk/internal/foreign/abi/ppc64/CallArranger.java

@@ -110,8 +110,8 @@ public abstract class CallArranger {
     public Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall, LinkerOptions options) {
         CallingSequenceBuilder csb = new CallingSequenceBuilder(C, forUpcall, options);
 
-        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true);
+        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess());
-        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false);
+        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false);
 
         boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout());
         if (returnInMemory) {
@@ -343,8 +343,11 @@ public abstract class CallArranger {
 
     // Compute recipe for transfering arguments / return values to C from Java.
     class UnboxBindingCalculator extends BindingCalculator {
-        UnboxBindingCalculator(boolean forArguments) {
+        private final boolean useAddressPairs;
+
+        UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) {
             super(forArguments);
+            this.useAddressPairs = useAddressPairs;
         }
 
         @Override
@@ -411,8 +414,16 @@ public abstract class CallArranger {
                 }
                 case POINTER -> {
                     VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER, false);
-                    bindings.unboxAddress()
+                    if (useAddressPairs) {
-                            .vmStore(storage, long.class);
+                        bindings.dup()
+                                .segmentBase()
+                                .vmStore(storage, Object.class)
+                                .segmentOffsetAllowHeap()
+                                .vmStore(null, long.class);
+                    } else {
+                        bindings.unboxAddress();
+                        bindings.vmStore(storage, long.class);
+                    }
                 }
                 case INTEGER -> {
                     // ABI requires all int types to get extended to 64 bit.

src/java.base/share/classes/jdk/internal/foreign/abi/riscv64/linux/LinuxRISCV64CallArranger.java

@@ -85,8 +85,8 @@ public class LinuxRISCV64CallArranger {
 
     public static Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall, LinkerOptions options) {
         CallingSequenceBuilder csb = new CallingSequenceBuilder(CLinux, forUpcall, options);
-        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true);
+        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess());
-        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false);
+        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false);
 
         boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout());
         if (returnInMemory) {
@@ -252,11 +252,13 @@ public class LinuxRISCV64CallArranger {
     }
 
     static final class UnboxBindingCalculator extends BindingCalculator {
-        final boolean forArguments;
+        protected final boolean forArguments;
+        private final boolean useAddressPairs;
 
-        UnboxBindingCalculator(boolean forArguments) {
+        UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) {
             super(forArguments);
             this.forArguments = forArguments;
+            this.useAddressPairs = useAddressPairs;
         }
 
         @Override
@@ -280,9 +282,17 @@ public class LinuxRISCV64CallArranger {
                     bindings.vmStore(storage, carrier);
                 }
                 case POINTER -> {
-                    bindings.unboxAddress();
                     VMStorage storage = storageCalculator.getStorage(StorageType.INTEGER);
-                    bindings.vmStore(storage, long.class);
+                    if (useAddressPairs) {
+                        bindings.dup()
+                                .segmentBase()
+                                .vmStore(storage, Object.class)
+                                .segmentOffsetAllowHeap()
+                                .vmStore(null, long.class);
+                    } else {
+                        bindings.unboxAddress();
+                        bindings.vmStore(storage, long.class);
+                    }
                 }
                 case STRUCT_REGISTER_X -> {
                     assert carrier == MemorySegment.class;

src/java.base/share/classes/jdk/internal/foreign/abi/s390/linux/LinuxS390CallArranger.java

@@ -82,8 +82,8 @@ public class LinuxS390CallArranger {
     public static Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall, LinkerOptions options) {
         CallingSequenceBuilder csb = new CallingSequenceBuilder(CLinux, forUpcall, options);
 
-        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true);
+        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess());
-        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false);
+        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false);
 
         boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout());
         if (returnInMemory) {
@@ -199,8 +199,11 @@ public class LinuxS390CallArranger {
 
     // Compute recipe for transferring arguments / return values to C from Java.
     static class UnboxBindingCalculator extends BindingCalculator {
-        UnboxBindingCalculator(boolean forArguments) {
+            private final boolean useAddressPairs;
+
+            UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) {
             super(forArguments);
+            this.useAddressPairs = useAddressPairs;
         }
 
         @Override
@@ -231,8 +234,16 @@ public class LinuxS390CallArranger {
                 }
                 case POINTER -> {
                     VMStorage storage = storageCalculator.getStorage(StorageType.INTEGER, false);
-                    bindings.unboxAddress()
+                    if (useAddressPairs) {
-                            .vmStore(storage, long.class);
+                        bindings.dup()
+                                .segmentBase()
+                                .vmStore(storage, Object.class)
+                                .segmentOffsetAllowHeap()
+                                .vmStore(null, long.class);
+                    } else {
+                        bindings.unboxAddress();
+                        bindings.vmStore(storage, long.class);
+                    }
                 }
                 case INTEGER -> {
                     // ABI requires all int types to get extended to 64 bit.

src/java.base/share/classes/jdk/internal/foreign/abi/x64/sysv/CallArranger.java

@@ -97,8 +97,8 @@ public class CallArranger {
     public static Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall, LinkerOptions options) {
         CallingSequenceBuilder csb = new CallingSequenceBuilder(CSysV, forUpcall, options);
 
-        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true);
+        BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess());
-        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false);
+        BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false);
 
         boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout());
         if (returnInMemory) {
@@ -246,9 +246,11 @@ public class CallArranger {
     }
 
     static class UnboxBindingCalculator extends BindingCalculator {
+        private final boolean useAddressPairs;
 
-        UnboxBindingCalculator(boolean forArguments) {
+        UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) {
             super(forArguments);
+            this.useAddressPairs = useAddressPairs;
         }
 
         @Override
@@ -275,10 +277,18 @@ public class CallArranger {
                     }
                 }
                 case POINTER -> {
-                    bindings.unboxAddress();
                     VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER);
-                    bindings.vmStore(storage, long.class);
+                    if (useAddressPairs) {
-                                    }
+                        bindings.dup()
+                                .segmentBase()
+                                .vmStore(storage, Object.class)
+                                .segmentOffsetAllowHeap()
+                                .vmStore(null, long.class);
+                    } else {
+                        bindings.unboxAddress();
+                        bindings.vmStore(storage, long.class);
+                    }
+                }
                 case INTEGER -> {
                     VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER);
                     bindings.vmStore(storage, carrier);

src/java.base/share/classes/jdk/internal/foreign/abi/x64/windows/CallArranger.java

@@ -87,9 +87,9 @@ public class CallArranger {
         class CallingSequenceBuilderHelper {
             final CallingSequenceBuilder csb = new CallingSequenceBuilder(CWindows, forUpcall, options);
             final BindingCalculator argCalc =
-                    forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true);
+                    forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess());
             final BindingCalculator retCalc =
-                    forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false);
+                    forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false);
 
             void addArgumentBindings(Class<?> carrier, MemoryLayout layout, boolean isVararg) {
                 csb.addArgumentBindings(carrier, layout, argCalc.getBindings(carrier, layout, isVararg));
@@ -184,9 +184,12 @@ public class CallArranger {
 
     static class UnboxBindingCalculator implements BindingCalculator {
         private final StorageCalculator storageCalculator;
+        private final boolean useAddressPairs;
 
-        UnboxBindingCalculator(boolean forArguments) {
+        UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) {
             this.storageCalculator = new StorageCalculator(forArguments);
+            assert !useAddressPairs || forArguments;
+            this.useAddressPairs = useAddressPairs;
         }
 
         @Override
@@ -209,9 +212,17 @@ public class CallArranger {
                     bindings.vmStore(storage, long.class);
                 }
                 case POINTER -> {
-                    bindings.unboxAddress();
                     VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER);
-                    bindings.vmStore(storage, long.class);
+                    if (useAddressPairs) {
+                        bindings.dup()
+                                .segmentBase()
+                                .vmStore(storage, Object.class)
+                                .segmentOffsetAllowHeap()
+                                .vmStore(null, long.class);
+                    } else {
+                        bindings.unboxAddress();
+                        bindings.vmStore(storage, long.class);
+                    }
                 }
                 case INTEGER -> {
                     VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER);

src/java.base/share/native/libfallbackLinker/fallbackLinker.c

@@ -28,6 +28,7 @@
 #include <ffi.h>
 
 #include <errno.h>
+#include <malloc.h>
 #include <stdint.h>
 #include <wchar.h>
 #ifdef _WIN64
@@ -109,9 +110,39 @@ static void do_capture_state(int32_t* value_ptr, int captured_state_mask) {
 }
 
 JNIEXPORT void JNICALL
-Java_jdk_internal_foreign_abi_fallback_LibFallback_doDowncall(JNIEnv* env, jclass cls, jlong cif, jlong fn, jlong rvalue, jlong avalues, jlong jcaptured_state, jint captured_state_mask) {
+Java_jdk_internal_foreign_abi_fallback_LibFallback_doDowncall(JNIEnv* env, jclass cls, jlong cif, jlong fn, jlong rvalue,
+                                                              jlong avalues, jlong jcaptured_state, jint captured_state_mask,
+                                                              jarray heapBases, jint numArgs) {
+  void** carrays;
+  if (heapBases != NULL) {
+    void** aptrs = jlong_to_ptr(avalues);
+    carrays = malloc(sizeof(void*) * numArgs);
+    for (int i = 0; i < numArgs; i++) {
+      jarray hb = (jarray) (*env)->GetObjectArrayElement(env, heapBases, i);
+      if (hb != NULL) {
+        // *(aptrs[i]) is the offset into the segment (from MS::address)
+        // we add the base address of the array to it here
+        jboolean isCopy;
+        jbyte* arrayPtr = (*env)->GetPrimitiveArrayCritical(env, hb, &isCopy);
+        carrays[i] = arrayPtr;
+        int offset = *((int*)aptrs[i]);
+        *((void**)aptrs[i]) = arrayPtr + offset;
+      }
+    }
+  }
+
   ffi_call(jlong_to_ptr(cif), jlong_to_ptr(fn), jlong_to_ptr(rvalue), jlong_to_ptr(avalues));
 
+  if (heapBases != NULL) {
+    for (int i = 0; i < numArgs; i++) {
+      jarray hb = (jarray) (*env)->GetObjectArrayElement(env, heapBases, i);
+      if (hb != NULL) {
+        (*env)->ReleasePrimitiveArrayCritical(env, hb, carrays[i], JNI_COMMIT);
+      }
+    }
+    free(carrays);
+  }
+
   if (captured_state_mask != 0) {
     int32_t* captured_state = jlong_to_ptr(jcaptured_state);
     do_capture_state(captured_state, captured_state_mask);

test/jdk/java/foreign/NativeTestHelper.java

@@ -195,7 +195,9 @@ public class NativeTestHelper {
         return result;
     }
 
-    public record TestValue (Object value, Consumer<Object> check) {}
+    public record TestValue (Object value, Consumer<Object> check) {
+        public void check(Object actual) { check.accept(actual); }
+    }
 
     public static TestValue genTestValue(MemoryLayout layout, SegmentAllocator allocator) {
         return genTestValue(DEFAULT_RANDOM, layout, allocator);
@@ -237,6 +239,9 @@ public class NativeTestHelper {
         } else if (layout instanceof ValueLayout.OfShort) {
             short value = (short) random.nextInt();
             return new TestValue(value, actual -> assertEquals(actual, value));
+        } else if (layout instanceof ValueLayout.OfChar) {
+            char value = (char) random.nextInt();
+            return new TestValue(value, actual -> assertEquals(actual, value));
         } else if (layout instanceof ValueLayout.OfInt) {
             int value = random.nextInt();
             return new TestValue(value, actual -> assertEquals(actual, value));

test/jdk/java/foreign/TestIllegalLink.java

@@ -102,7 +102,7 @@ public class TestIllegalLink extends NativeTestHelper {
         return new Object[][]{
             { Linker.Option.firstVariadicArg(0) },
             { Linker.Option.captureCallState("errno") },
-            { Linker.Option.critical() },
+            { Linker.Option.critical(false) },
         };
     }
 
@@ -194,7 +194,7 @@ public class TestIllegalLink extends NativeTestHelper {
             },
             {
                     FunctionDescriptor.ofVoid(),
-                    new Linker.Option[]{Linker.Option.critical(), Linker.Option.captureCallState("errno")},
+                    new Linker.Option[]{Linker.Option.critical(false), Linker.Option.captureCallState("errno")},
                     "Incompatible linker options: captureCallState, critical"
             },
         }));

test/jdk/java/foreign/critical/TestCritical.java

@@ -0,0 +1,182 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @library ../ /test/lib
+ *
+ * @run testng/othervm --enable-native-access=ALL-UNNAMED TestCritical
+ */
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.Linker;
+import java.lang.foreign.MemoryLayout;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.SegmentAllocator;
+import java.lang.foreign.SequenceLayout;
+import java.lang.foreign.StructLayout;
+import java.lang.foreign.ValueLayout;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.VarHandle;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.IntFunction;
+import java.util.stream.Stream;
+
+import static org.testng.Assert.assertEquals;
+
+public class TestCritical extends NativeTestHelper {
+
+    static {
+        System.loadLibrary("Critical");
+    }
+
+    @Test
+    public void testEmpty() throws Throwable {
+        MethodHandle handle = downcallHandle("empty", FunctionDescriptor.ofVoid(), Linker.Option.critical(false));
+        handle.invokeExact();
+    }
+
+    @Test
+    public void testIdentity() throws Throwable {
+        MethodHandle handle = downcallHandle("identity", FunctionDescriptor.of(C_INT, C_INT), Linker.Option.critical(false));
+        int result = (int) handle.invokeExact(42);
+        assertEquals(result, 42);
+    }
+
+    @Test
+    public void testWithReturnBuffer() throws Throwable {
+        StructLayout bigLayout = MemoryLayout.structLayout(
+                C_LONG_LONG.withName("x"),
+                C_LONG_LONG.withName("y"));
+
+        MethodHandle handle = downcallHandle("with_return_buffer", FunctionDescriptor.of(bigLayout), Linker.Option.critical(false));
+        VarHandle vhX = bigLayout.varHandle(MemoryLayout.PathElement.groupElement("x"));
+        VarHandle vhY = bigLayout.varHandle(MemoryLayout.PathElement.groupElement("y"));
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment result = (MemorySegment) handle.invokeExact((SegmentAllocator) arena);
+            long x = (long) vhX.get(result, 0L);
+            assertEquals(x, 10);
+            long y = (long) vhY.get(result, 0L);
+            assertEquals(y, 11);
+        }
+    }
+
+    public record AllowHeapCase(IntFunction<MemorySegment> newArraySegment, ValueLayout elementLayout,
+                                String fName, FunctionDescriptor fDesc, boolean readOnly) {}
+
+    @Test(dataProvider = "allowHeapCases")
+    public void testAllowHeap(AllowHeapCase testCase) throws Throwable {
+        MethodHandle handle = downcallHandle(testCase.fName(), testCase.fDesc(), Linker.Option.critical(true));
+        int elementCount = 10;
+        MemorySegment heapSegment = testCase.newArraySegment().apply(elementCount);
+        if (testCase.readOnly()) {
+            heapSegment = heapSegment.asReadOnly();
+        }
+        SequenceLayout sequence = MemoryLayout.sequenceLayout(elementCount, testCase.elementLayout());
+
+        try (Arena arena = Arena.ofConfined()) {
+            TestValue[] tvs = genTestArgs(testCase.fDesc(), arena);
+            Object[] args = Stream.of(tvs).map(TestValue::value).toArray();
+
+            // inject our custom last three arguments
+            args[args.length - 1] = (int) sequence.byteSize();
+            TestValue sourceSegment = genTestValue(sequence, arena);
+            args[args.length - 2] = sourceSegment.value();
+            args[args.length - 3] = heapSegment;
+
+            if (handle.type().parameterType(0) == SegmentAllocator.class) {
+                Object[] newArgs = new Object[args.length + 1];
+                newArgs[0] = arena;
+                System.arraycopy(args, 0, newArgs, 1, args.length);
+                args = newArgs;
+            }
+
+            Object o = handle.invokeWithArguments(args);
+
+            if (o != null) {
+                tvs[0].check(o);
+            }
+
+            // check that writes went through to array
+            sourceSegment.check(heapSegment);
+        }
+    }
+
+    @DataProvider
+    public Object[][] allowHeapCases() {
+        FunctionDescriptor voidDesc = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT);
+        FunctionDescriptor intDesc = voidDesc.changeReturnLayout(C_INT).insertArgumentLayouts(0, C_INT);
+        StructLayout L2 = MemoryLayout.structLayout(
+            C_LONG_LONG.withName("x"),
+            C_LONG_LONG.withName("y")
+        );
+        FunctionDescriptor L2Desc = voidDesc.changeReturnLayout(L2).insertArgumentLayouts(0, L2);
+        StructLayout L3 = MemoryLayout.structLayout(
+            C_LONG_LONG.withName("x"),
+            C_LONG_LONG.withName("y"),
+            C_LONG_LONG.withName("z")
+        );
+        FunctionDescriptor L3Desc = voidDesc.changeReturnLayout(L3).insertArgumentLayouts(0, L3);
+        FunctionDescriptor stackDesc = voidDesc.insertArgumentLayouts(0,
+                C_LONG_LONG, C_LONG_LONG, C_LONG_LONG, C_LONG_LONG,
+                C_LONG_LONG, C_LONG_LONG, C_LONG_LONG, C_LONG_LONG,
+                C_CHAR, C_SHORT, C_INT);
+
+        List<AllowHeapCase> cases = new ArrayList<>();
+
+        for (HeapSegmentFactory hsf : HeapSegmentFactory.values()) {
+            cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_void", voidDesc, false));
+            cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_int", intDesc, false));
+            cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_return_buffer", L2Desc, false));
+            cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_imr", L3Desc, false));
+            cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_void_stack", stackDesc, false));
+            // readOnly
+            cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_void", voidDesc, true));
+        }
+
+        return cases.stream().map(e -> new Object[]{ e }).toArray(Object[][]::new);
+    }
+
+    private enum HeapSegmentFactory {
+        BYTE(i -> MemorySegment.ofArray(new byte[i]), ValueLayout.JAVA_BYTE),
+        SHORT(i -> MemorySegment.ofArray(new short[i]), ValueLayout.JAVA_SHORT),
+        CHAR(i -> MemorySegment.ofArray(new char[i]), ValueLayout.JAVA_CHAR),
+        INT(i -> MemorySegment.ofArray(new int[i]), ValueLayout.JAVA_INT),
+        LONG(i -> MemorySegment.ofArray(new long[i]), ValueLayout.JAVA_LONG),
+        FLOAT(i -> MemorySegment.ofArray(new float[i]), ValueLayout.JAVA_FLOAT),
+        DOUBLE(i -> MemorySegment.ofArray(new double[i]), ValueLayout.JAVA_DOUBLE);
+
+        IntFunction<MemorySegment> newArray;
+        ValueLayout elementLayout;
+
+        private HeapSegmentFactory(IntFunction<MemorySegment> newArray, ValueLayout elementLayout) {
+            this.newArray = newArray;
+            this.elementLayout = elementLayout;
+        }
+    }
+}

test/jdk/java/foreign/critical/TestCriticalUpcall.java

@@ -50,7 +50,7 @@ public class TestCriticalUpcall extends UpcallTestHelper {
         public static void main(String[] args) throws Throwable {
             System.loadLibrary("Critical");
 
-            MethodHandle mh = downcallHandle("do_upcall", FunctionDescriptor.ofVoid(C_POINTER), Linker.Option.critical());
+            MethodHandle mh = downcallHandle("do_upcall", FunctionDescriptor.ofVoid(C_POINTER), Linker.Option.critical(false));
             MemorySegment stub = upcallStub(Runner.class, "target", FunctionDescriptor.ofVoid());
             mh.invokeExact(stub);
         }

test/jdk/java/foreign/critical/TestStressAllowHeap.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @library ../ /test/lib
+ * @requires vm.debug
+ * @run main/othervm
+ *   -Xms1g -Xmx1g
+ *   -XX:+CheckUnhandledOops
+ *   -Xlog:gc -Xlog:gc+jni=debug
+ *   --enable-native-access=ALL-UNNAMED
+ *   TestStressAllowHeap
+ */
+
+import java.lang.foreign.*;
+import java.lang.invoke.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static jdk.test.lib.Asserts.*;
+
+/**
+ * Test verifies the GCLocker::lock_critical slow path with FFM.
+ * This is the case where we enter a critical section with _needs_gc == true,
+ * and need to take a safepoint.
+ *
+ * Based on gc/TestJNICriticalStressTest
+ */
+public class TestStressAllowHeap {
+    private static final long DURATION_SECONDS = 30;
+
+    public static void main(String... args) throws Exception {
+        System.out.println("Running for " + DURATION_SECONDS + " secs");
+
+        int numCriticalThreads = Runtime.getRuntime().availableProcessors();
+        System.out.println("Starting " + numCriticalThreads + " critical threads");
+        for (int i = 0; i < numCriticalThreads; i += 1) {
+            Thread.ofPlatform().start(new CriticalWorker());
+        }
+
+        long durationMS = 1000L * DURATION_SECONDS;
+        try {
+            Thread.sleep(durationMS);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+            System.exit(-1);
+        }
+
+        // hitting the problematic code path doesn't seem to be guaranteed
+        // and we can not guarantee it by, e.g. stalling in a critical method
+        // since that can lock up the VM (and our test code)
+    }
+
+    private static class CriticalWorker extends NativeTestHelper implements Runnable {
+        static {
+            System.loadLibrary("Critical");
+        }
+
+        private void doStep(MethodHandle handle, SequenceLayout sequence) throws Throwable {
+            try (Arena arena = Arena.ofConfined()) {
+                MemorySegment heapSegment = MemorySegment.ofArray(new int[(int) sequence.elementCount()]);
+                TestValue sourceSegment = genTestValue(sequence, arena);
+
+                handle.invokeExact(heapSegment, (MemorySegment) sourceSegment.value(), (int) sequence.byteSize());
+
+                // check that writes went through to array
+                sourceSegment.check(heapSegment);
+            }
+        }
+
+        @Override
+        public void run() {
+            FunctionDescriptor fDesc = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT);
+            MethodHandle handle = Linker.nativeLinker().downcallHandle(
+                SymbolLookup.loaderLookup().find("test_allow_heap_void").get(),
+                fDesc,
+                Linker.Option.critical(true));
+            int elementCount = 10;
+            SequenceLayout sequence = MemoryLayout.sequenceLayout(elementCount, C_INT);
+            while (true) {
+                try {
+                    doStep(handle, sequence);
+                } catch (Throwable t) {
+                    throw new RuntimeException(t);
+                }
+            }
+        }
+    }
+}

test/jdk/java/foreign/critical/libCritical.c

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <errno.h>
+
+#ifdef _WIN64
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+EXPORT void empty() {}
+
+EXPORT int identity(int value) {
+    return value;
+}
+
+// 128 bit struct returned in buffer on SysV
+struct Big {
+    long long x;
+    long long y;
+};
+
+EXPORT struct Big with_return_buffer() {
+    struct Big b;
+    b.x = 10;
+    b.y = 11;
+    return b;
+}
+
+EXPORT void do_upcall(void(*f)(void)) {
+    f();
+}
+
+// copy bytes into heap array
+EXPORT void test_allow_heap_void(unsigned char* heapArr, unsigned char* nativeArr, int numBytes) {
+    for (int i = 0; i < numBytes; i++) {
+        heapArr[i] = nativeArr[i];
+    }
+}
+
+EXPORT int test_allow_heap_int(int a0, unsigned char* heapArr, unsigned char* nativeArr, int numBytes) {
+    for (int i = 0; i < numBytes; i++) {
+        heapArr[i] = nativeArr[i];
+    }
+    return a0;
+}
+
+struct L2 {
+    long long x;
+    long long y;
+};
+
+EXPORT struct L2 test_allow_heap_return_buffer(struct L2 a0, unsigned char* heapArr, unsigned char* nativeArr, int numBytes) {
+    for (int i = 0; i < numBytes; i++) {
+        heapArr[i] = nativeArr[i];
+    }
+    return a0;
+}
+
+struct L3 {
+    long long x;
+    long long y;
+    long long z;
+};
+
+EXPORT struct L3 test_allow_heap_imr(struct L3 a0, unsigned char* heapArr, unsigned char* nativeArr, int numBytes) {
+    for (int i = 0; i < numBytes; i++) {
+        heapArr[i] = nativeArr[i];
+    }
+    return a0;
+}
+
+// copy bytes into heap array
+EXPORT void test_allow_heap_void_stack(long long a0, long long a1, long long a2, long long a3, long long a4, long long a5,
+                                       long long a6, long long a7, char c0, short s0, int i0,
+                                       unsigned char* heapArr, unsigned char* nativeArr, int numBytes) {
+    for (int i = 0; i < numBytes; i++) {
+        heapArr[i] = nativeArr[i];
+    }
+}

test/jdk/java/foreign/largestub/TestLargeStub.java

@@ -36,20 +36,37 @@ import java.lang.foreign.MemoryLayout;
 import java.util.stream.Stream;
 
 public class TestLargeStub extends NativeTestHelper {
+
+    MemoryLayout STRUCT_LL = MemoryLayout.structLayout(
+        C_LONG_LONG,
+        C_LONG_LONG
+    ); // 16 byte struct triggers return buffer usage on SysV
+
     @Test
     public void testDowncall() {
         // Link a handle with a large number of arguments, to try and overflow the code buffer
         Linker.nativeLinker().downcallHandle(
-                FunctionDescriptor.of(C_LONG_LONG,
+                FunctionDescriptor.of(STRUCT_LL,
-                        Stream.generate(() -> C_DOUBLE).limit(50).toArray(MemoryLayout[]::new)),
+                        Stream.generate(() -> C_DOUBLE).limit(124).toArray(MemoryLayout[]::new)),
                 Linker.Option.captureCallState("errno"));
     }
 
+    @Test
+    public void testDowncallAllowHeap() {
+        // Link a handle with a large number of address arguments, to try and overflow the code buffer
+        // Using 83 parameters should get us 255 parameter slots in total:
+        // 83 oops + 166 for offsets + 2 for the target address + 2 for return buffer + MH recv. + NEP
+        Linker.nativeLinker().downcallHandle(
+                FunctionDescriptor.of(STRUCT_LL,
+                        Stream.generate(() -> C_POINTER).limit(83).toArray(MemoryLayout[]::new)),
+                Linker.Option.critical(true));
+    }
+
     @Test
     public void testUpcall() {
         // Link a handle with a large number of arguments, to try and overflow the code buffer
         Linker.nativeLinker().downcallHandle(
-                FunctionDescriptor.of(C_LONG_LONG,
+                FunctionDescriptor.of(STRUCT_LL,
-                        Stream.generate(() -> C_DOUBLE).limit(50).toArray(MemoryLayout[]::new)));
+                        Stream.generate(() -> C_DOUBLE).limit(125).toArray(MemoryLayout[]::new)));
     }
 }

test/jdk/java/foreign/trivial/TestCritical.java

@@ -1,81 +0,0 @@
-/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-/*
- * @test
- * @library ../ /test/lib
- * @run testng/othervm --enable-native-access=ALL-UNNAMED TestCritical
- */
-
-import org.testng.annotations.Test;
-
-import java.lang.foreign.Arena;
-import java.lang.foreign.FunctionDescriptor;
-import java.lang.foreign.Linker;
-import java.lang.foreign.MemoryLayout;
-import java.lang.foreign.MemorySegment;
-import java.lang.foreign.SegmentAllocator;
-import java.lang.foreign.StructLayout;
-import java.lang.invoke.MethodHandle;
-import java.lang.invoke.VarHandle;
-
-import static org.testng.Assert.assertEquals;
-
-public class TestCritical extends NativeTestHelper {
-
-    static {
-        System.loadLibrary("Critical");
-    }
-
-    @Test
-    public void testEmpty() throws Throwable {
-        MethodHandle handle = downcallHandle("empty", FunctionDescriptor.ofVoid(), Linker.Option.critical());
-        handle.invokeExact();
-    }
-
-    @Test
-    public void testIdentity() throws Throwable {
-        MethodHandle handle = downcallHandle("identity", FunctionDescriptor.of(C_INT, C_INT), Linker.Option.critical());
-        int result = (int) handle.invokeExact(42);
-        assertEquals(result, 42);
-    }
-
-    @Test
-    public void testWithReturnBuffer() throws Throwable {
-        StructLayout bigLayout = MemoryLayout.structLayout(
-                C_LONG_LONG.withName("x"),
-                C_LONG_LONG.withName("y"));
-
-        MethodHandle handle = downcallHandle("with_return_buffer", FunctionDescriptor.of(bigLayout), Linker.Option.critical());
-        VarHandle vhX = bigLayout.varHandle(MemoryLayout.PathElement.groupElement("x"));
-        VarHandle vhY = bigLayout.varHandle(MemoryLayout.PathElement.groupElement("y"));
-        try (Arena arena = Arena.ofConfined()) {
-            MemorySegment result = (MemorySegment) handle.invokeExact((SegmentAllocator) arena);
-            long x = (long) vhX.get(result, 0L);
-            assertEquals(x, 10);
-            long y = (long) vhY.get(result, 0L);
-            assertEquals(y, 11);
-        }
-    }
-
-}

test/micro/org/openjdk/bench/java/lang/foreign/CallOverheadHelper.java

@@ -113,7 +113,7 @@ public class CallOverheadHelper extends CLayouts {
             MethodType mt = MethodType.methodType(void.class);
             FunctionDescriptor fd = FunctionDescriptor.ofVoid();
             func_v = abi.downcallHandle(fd);
-            func_critical_v = abi.downcallHandle(fd, Linker.Option.critical());
+            func_critical_v = abi.downcallHandle(fd, Linker.Option.critical(false));
             func = insertArguments(func_v, 0, func_addr);
             func_critical = insertArguments(func_critical_v, 0, func_addr);
         }
@@ -121,7 +121,7 @@ public class CallOverheadHelper extends CLayouts {
             identity_addr = loaderLibs.find("identity").orElseThrow();
             FunctionDescriptor fd = FunctionDescriptor.of(C_INT, C_INT);
             identity_v = abi.downcallHandle(fd);
-            identity_critical_v = abi.downcallHandle(fd, Linker.Option.critical());
+            identity_critical_v = abi.downcallHandle(fd, Linker.Option.critical(false));
             identity = insertArguments(identity_v, 0, identity_addr);
             identity_critical = insertArguments(identity_critical_v, 0, identity_addr);
         }

test/micro/org/openjdk/bench/java/lang/foreign/CriticalCalls.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.bench.java.lang.foreign;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.Linker;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.SegmentAllocator;
+import java.lang.foreign.SymbolLookup;
+
+import java.lang.invoke.MethodHandle;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+import static java.lang.foreign.ValueLayout.*;
+
+@BenchmarkMode(Mode.AverageTime)
+@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@State(org.openjdk.jmh.annotations.Scope.Thread)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Fork(value = 3, jvmArgsAppend = { "--enable-native-access=ALL-UNNAMED" })
+public class CriticalCalls {
+
+    static final MethodHandle PINNED;
+    static final MethodHandle NOT_PINNED;
+
+    static {
+        System.loadLibrary("CriticalCalls");
+        SymbolLookup lookup = SymbolLookup.loaderLookup();
+
+        MemorySegment sumIntsSym = lookup.find("sum_ints").get();
+        FunctionDescriptor sumIntsDesc = FunctionDescriptor.of(JAVA_INT, ADDRESS, JAVA_INT);
+
+        PINNED = Linker.nativeLinker().downcallHandle(
+            sumIntsSym,
+            sumIntsDesc,
+            Linker.Option.critical(true));
+        NOT_PINNED = Linker.nativeLinker().downcallHandle(
+            sumIntsSym,
+            sumIntsDesc,
+            Linker.Option.critical(false));
+    }
+
+    @Param({"100", "10000", "1000000"})
+    int size;
+
+    int[] arr;
+    SegmentAllocator recycler;
+
+    @Setup
+    public void setup() {
+        arr = new int[size];
+        for (int i = 0; i < size; i++) {
+            arr[i] = i;
+        }
+
+        recycler = SegmentAllocator.prefixAllocator(Arena.ofAuto().allocate(JAVA_INT, arr.length));
+    }
+
+    @Benchmark
+    public int callPinned() throws Throwable {
+        return (int) PINNED.invokeExact(MemorySegment.ofArray(arr), arr.length);
+    }
+
+    @Benchmark
+    public int callNotPinned() throws Throwable {
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment nativeArr = arena.allocateFrom(JAVA_INT, arr);
+            return (int) NOT_PINNED.invokeExact(nativeArr, arr.length);
+        }
+    }
+
+    @Benchmark
+    public int callRecycled() throws Throwable {
+        MemorySegment nativeArr = recycler.allocateFrom(JAVA_INT, arr);
+        return (int) NOT_PINNED.invokeExact(nativeArr, arr.length);
+    }
+}

test/micro/org/openjdk/bench/java/lang/foreign/Utils.java

@@ -28,7 +28,7 @@ import java.lang.reflect.Field;
 
 public class Utils {
 
-    static final Unsafe unsafe;
+    public static final Unsafe unsafe;
 
     //setup unsafe
     static {

test/micro/org/openjdk/bench/java/lang/foreign/libCriticalCalls.c

@@ -21,33 +21,16 @@
  * questions.
  */
 
-#include <errno.h>
-
 #ifdef _WIN64
 #define EXPORT __declspec(dllexport)
 #else
 #define EXPORT
 #endif
 
-EXPORT void empty() {}
+EXPORT int sum_ints(int* arr, int size) {
-
+  int sum = 0;
-EXPORT int identity(int value) {
+  for (int i = 0; i < size; i++) {
-    return value;
+    sum += arr[i];
-}
+  }
-
+  return sum;
-// 128 bit struct returned in buffer on SysV
-struct Big {
-    long long x;
-    long long y;
-};
-
-EXPORT struct Big with_return_buffer() {
-    struct Big b;
-    b.x = 10;
-    b.y = 11;
-    return b;
-}
-
-EXPORT void do_upcall(void(*f)(void)) {
-    f();
 }

test/micro/org/openjdk/bench/java/lang/foreign/libToCString.c

@@ -26,6 +26,8 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "jlong.h"
+
 JNIEXPORT jlong JNICALL Java_org_openjdk_bench_java_lang_foreign_ToCStringTest_writeString(JNIEnv *const env, const jclass cls, const jstring text) {
     const char *str = (*env)->GetStringUTFChars(env, text, NULL);
     jlong addr = ptr_to_jlong(str);

test/micro/org/openjdk/bench/java/lang/foreign/libToJavaString.c

@@ -28,6 +28,8 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "jlong.h"
+
 JNIEXPORT jstring JNICALL Java_org_openjdk_bench_java_lang_foreign_ToJavaStringTest_readString(JNIEnv *const env, const jclass cls, jlong addr) {
     return (*env)->NewStringUTF(env, jlong_to_ptr(addr));
 }

test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayCriticalXorOpImpl.java

@@ -0,0 +1,11 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+public class GetArrayCriticalXorOpImpl implements XorOp {
+
+    static {
+        System.loadLibrary("jnitest");
+    }
+
+    // Uses {Get|Release}PrimitiveArrayCritical to access the byte arrays
+    public native void xor(byte[] src, int sOff, byte[] dst, int dOff, int len);
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayElementsXorOpImpl.java

@@ -0,0 +1,11 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+public class GetArrayElementsXorOpImpl implements XorOp {
+
+    static {
+        System.loadLibrary("jnitest");
+    }
+
+    // Uses {Get|Release}ByteArrayElements to access the byte arrays
+    public native void xor(byte[] src, int sOff, byte[] dst, int dOff, int len);
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpCriticalImpl.java

@@ -0,0 +1,30 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.Linker;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.SymbolLookup;
+import java.lang.invoke.MethodHandle;
+
+import static java.lang.foreign.Linker.Option.critical;
+import static org.openjdk.bench.java.lang.foreign.CLayouts.*;
+
+public class GetArrayForeignXorOpCriticalImpl implements XorOp {
+
+    static {
+        System.loadLibrary("jnitest");
+
+        Linker linker;
+        linker = Linker.nativeLinker();
+        FunctionDescriptor xor_op_func = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT);
+        xor_op = linker.downcallHandle(SymbolLookup.loaderLookup().find("xor_op").orElseThrow(), xor_op_func, critical(true));
+    }
+
+    static final MethodHandle xor_op;
+    GetArrayForeignXorOpCriticalImpl() {
+    }
+
+    public void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable {
+        xor_op.invokeExact(MemorySegment.ofArray(src).asSlice(sOff), MemorySegment.ofArray(dst).asSlice(dOff), len);
+    }
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpImpl.java

@@ -0,0 +1,37 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.Linker;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.SymbolLookup;
+import java.lang.invoke.MethodHandle;
+
+import static java.lang.foreign.Linker.Option.critical;
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.openjdk.bench.java.lang.foreign.CLayouts.*;
+
+public class GetArrayForeignXorOpImpl implements XorOp {
+
+    static {
+        System.loadLibrary("jnitest");
+
+        Linker linker;
+        linker = Linker.nativeLinker();
+        FunctionDescriptor xor_op_func = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT);
+        xor_op = linker.downcallHandle(SymbolLookup.loaderLookup().find("xor_op").orElseThrow(), xor_op_func, critical(false));
+    }
+
+    static final MethodHandle xor_op;
+    GetArrayForeignXorOpImpl() {
+    }
+
+    public void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable {
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment srcBuf = arena.allocateFrom(JAVA_BYTE, MemorySegment.ofArray(src), JAVA_BYTE, sOff, len);
+            MemorySegment dstBuf = arena.allocateFrom(JAVA_BYTE, MemorySegment.ofArray(dst), JAVA_BYTE, dOff, len);
+            xor_op.invokeExact(srcBuf, dstBuf, len);
+            MemorySegment.copy(dstBuf, JAVA_BYTE, 0, dst, dOff, len);
+        }
+    }
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpInitImpl.java

@@ -0,0 +1,40 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.Linker;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.SymbolLookup;
+import java.lang.invoke.MethodHandle;
+
+import static java.lang.foreign.Linker.Option.critical;
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.openjdk.bench.java.lang.foreign.CLayouts.C_INT;
+import static org.openjdk.bench.java.lang.foreign.CLayouts.C_POINTER;
+
+public class GetArrayForeignXorOpInitImpl implements XorOp {
+
+    static {
+        System.loadLibrary("jnitest");
+
+        Linker linker;
+        linker = Linker.nativeLinker();
+        FunctionDescriptor xor_op_func = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT);
+        xor_op = linker.downcallHandle(SymbolLookup.loaderLookup().find("xor_op").orElseThrow(), xor_op_func, critical(false));
+    }
+
+    static final MethodHandle xor_op;
+    GetArrayForeignXorOpInitImpl() {
+    }
+
+    public void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable {
+        try (Arena arena = Arena.ofConfined()) {
+            MemorySegment srcBuf = arena.allocate(len);
+            MemorySegment.copy(src, sOff, srcBuf, JAVA_BYTE, 0, len);
+            MemorySegment dstBuf = arena.allocate(len);
+            MemorySegment.copy(dst, dOff, dstBuf, JAVA_BYTE, 0, len);
+            xor_op.invokeExact(srcBuf, dstBuf, len);
+            MemorySegment.copy(dstBuf, JAVA_BYTE, 0, dst, dOff, len);
+        }
+    }
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayRegionXorOpImpl.java

@@ -0,0 +1,11 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+public class GetArrayRegionXorOpImpl implements XorOp {
+
+    static {
+        System.loadLibrary("jnitest");
+    }
+
+    // Uses {Get|Set}ByteArrayRegion to access the byte arrays
+    public native void xor(byte[] src, int sOff, byte[] dst, int dOff, int len);
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayUnsafeXorOpImpl.java

@@ -0,0 +1,47 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+import org.openjdk.bench.java.lang.foreign.Utils;
+import sun.misc.Unsafe;
+
+import java.lang.foreign.Arena;
+import java.lang.foreign.FunctionDescriptor;
+import java.lang.foreign.Linker;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.SymbolLookup;
+import java.lang.invoke.MethodHandle;
+
+import static java.lang.foreign.Linker.Option.critical;
+import static java.lang.foreign.ValueLayout.JAVA_BYTE;
+import static org.openjdk.bench.java.lang.foreign.CLayouts.*;
+
+public class GetArrayUnsafeXorOpImpl implements XorOp {
+
+    static final Unsafe UNSAFE = Utils.unsafe;
+    static final int BYTE_ARR_OFFSET = Utils.unsafe.arrayBaseOffset(byte[].class);
+
+    static {
+        System.loadLibrary("jnitest");
+
+        Linker linker;
+        linker = Linker.nativeLinker();
+        FunctionDescriptor xor_op_func = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT);
+        xor_op = linker.downcallHandle(SymbolLookup.loaderLookup().find("xor_op").orElseThrow(), xor_op_func, critical(false));
+    }
+
+    static final MethodHandle xor_op;
+    GetArrayUnsafeXorOpImpl() {
+    }
+
+    public void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable {
+        long srcBuf = UNSAFE.allocateMemory(len);
+        long dstBuf = UNSAFE.allocateMemory(len);
+        UNSAFE.copyMemory(src, sOff + BYTE_ARR_OFFSET, null, srcBuf, len);
+        UNSAFE.copyMemory(dst, dOff + BYTE_ARR_OFFSET, null, dstBuf, len);
+        xorOp(srcBuf, dstBuf, len);
+        UNSAFE.copyMemory(null, dstBuf, dst, dOff + BYTE_ARR_OFFSET, len);
+        UNSAFE.freeMemory(srcBuf);
+        UNSAFE.freeMemory(dstBuf);
+    }
+
+    native void xorOp(long src, long dst, int len);
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/XorOp.java

@@ -0,0 +1,7 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+public interface XorOp {
+
+    void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable;
+
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/XorTest.java

@@ -0,0 +1,130 @@
+package org.openjdk.bench.java.lang.foreign.xor;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS)
+@State(org.openjdk.jmh.annotations.Scope.Thread)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Fork(value = 3, jvmArgsAppend = { "--enable-native-access=ALL-UNNAMED" })
+
+public class XorTest {
+
+    XorOp impl = null;
+    int alen;
+    int off;
+    int len;
+    byte[] src, dst;
+
+    @Param
+    SizeKind sizeKind;
+
+    @Param
+    ArrayKind arrayKind;
+
+    public enum SizeKind {
+        SMALL,
+        MEDIUM,
+        LARGE;
+    }
+
+    public enum ArrayKind {
+        JNI_ELEMENTS,
+        JNI_REGION,
+        JNI_CRITICAL,
+        FOREIGN_NO_INIT,
+        FOREIGN_INIT,
+        FOREIGN_CRITICAL,
+        UNSAFE;
+    }
+
+    @Setup
+    public void setup() throws Throwable {
+        switch (arrayKind) {
+            case JNI_CRITICAL:
+                impl = new GetArrayCriticalXorOpImpl();
+                break;
+            case JNI_ELEMENTS:
+                impl = new GetArrayElementsXorOpImpl();
+                break;
+            case JNI_REGION:
+                impl = new GetArrayRegionXorOpImpl();
+                break;
+            case FOREIGN_NO_INIT:
+                impl = new GetArrayForeignXorOpImpl();
+                break;
+            case FOREIGN_INIT:
+                impl = new GetArrayForeignXorOpInitImpl();
+                break;
+            case FOREIGN_CRITICAL:
+                impl = new GetArrayForeignXorOpCriticalImpl();
+                break;
+            case UNSAFE:
+                impl = new GetArrayUnsafeXorOpImpl();
+                break;
+            default:
+                throw new UnsupportedOperationException(arrayKind.toString());
+        }
+
+        switch (sizeKind) {
+            case SMALL:
+                alen = 1048576;             // 1 MB
+                off = 1024 * 10;
+                len = 1024 * 100;           // 100 KB
+                break;
+            case MEDIUM:
+                alen = 1048576 * 8;         // 8 MB
+                off = 1048576 * 1;
+                len = 1048576 * 2;          // 2 MB
+                break;
+            case LARGE:
+                alen = 1048576 * 100;       // 100 MB
+                off = 1048576 * 5;
+                len = 1048576 * 10;         // 10 MB
+                break;
+            default:
+                throw new UnsupportedOperationException(sizeKind.toString());
+        }
+
+        src = new byte[alen];
+        dst = new byte[alen];
+        Arrays.fill(src, off, off + len, (byte)0xaa);
+        Arrays.fill(dst, off, off + len, (byte)0x5a);
+        check();
+    }
+
+    void check() throws Throwable {
+        impl.xor(src, off, dst, off, len);
+        if (!verify(dst, off, off + len, (byte)0xf0)) {
+            throw new IllegalStateException("Failed to verify");
+        }
+    }
+
+
+    @Benchmark
+    public void xor() throws Throwable {
+        impl.xor(src, off, dst, off, len);
+    }
+
+    static boolean verify(byte[] buf, int start, int end, byte val) {
+        for (int i = start; i < end; ++i) {
+            if (buf[i] != val)
+                return false;
+        }
+        return true;
+    }
+
+}

test/micro/org/openjdk/bench/java/lang/foreign/xor/libjnitest.c

@@ -0,0 +1,75 @@
+#include <stdlib.h>
+#include <jni.h>
+
+JNIEXPORT void xor_op(jbyte *restrict src, jbyte *restrict dst, jint len) {
+    for (int i = 0; i < len; ++i) {
+        dst[i] ^= src[i];
+    }
+}
+
+/*
+ * Class:     com_oracle_jnitest_GetArrayCriticalXorOpImpl
+ * Method:    xor
+ * Signature: ([BI[BII)V
+ */
+JNIEXPORT void JNICALL Java_org_openjdk_bench_java_lang_foreign_xor_GetArrayCriticalXorOpImpl_xor
+  (JNIEnv *env, jobject obj, jbyteArray src, jint sOff, jbyteArray dst, jint dOff, jint len) {
+    jbyte *sbuf = NULL;
+    jbyte *dbuf = NULL;
+    jboolean sIsCopy = JNI_FALSE;
+    jboolean dIsCopy = JNI_FALSE;
+
+    sbuf = (*env)->GetPrimitiveArrayCritical(env, src, &sIsCopy);
+    dbuf = (*env)->GetPrimitiveArrayCritical(env, dst, &dIsCopy);
+    xor_op(&sbuf[sOff], &dbuf[dOff], len);
+    (*env)->ReleasePrimitiveArrayCritical(env, dst, dbuf, 0);
+    (*env)->ReleasePrimitiveArrayCritical(env, src, sbuf, JNI_ABORT);
+}
+
+/*
+ * Class:     com_oracle_jnitest_GetArrayElementsXorOpImpl
+ * Method:    xor
+ * Signature: ([BI[BII)V
+ */
+JNIEXPORT void JNICALL Java_org_openjdk_bench_java_lang_foreign_xor_GetArrayElementsXorOpImpl_xor
+  (JNIEnv *env, jobject obj, jbyteArray src, jint sOff, jbyteArray dst, jint dOff, jint len) {
+    jbyte *sbuf = NULL;
+    jbyte *dbuf = NULL;
+    jboolean sIsCopy = JNI_FALSE;
+    jboolean dIsCopy = JNI_FALSE;
+
+    sbuf = (*env)->GetByteArrayElements(env, src, &sIsCopy);
+    dbuf = (*env)->GetByteArrayElements(env, dst, &dIsCopy);
+    xor_op(&sbuf[sOff], &dbuf[dOff], len);
+    (*env)->ReleaseByteArrayElements(env, dst, dbuf, 0);
+    (*env)->ReleaseByteArrayElements(env, src, sbuf, JNI_ABORT);
+}
+
+/*
+ * Class:     com_oracle_jnitest_GetArrayRegionXorOpImpl
+ * Method:    xor
+ * Signature: ([BI[BII)V
+ */
+JNIEXPORT void JNICALL Java_org_openjdk_bench_java_lang_foreign_xor_GetArrayRegionXorOpImpl_xor
+  (JNIEnv *env, jobject obj, jbyteArray src, jint sOff, jbyteArray dst, jint dOff, jint len) {
+    jbyte *sbuf = NULL;
+    jbyte *dbuf = NULL;
+
+    sbuf = malloc(len);
+    dbuf = malloc(len);
+
+    (*env)->GetByteArrayRegion(env, src, sOff, len, sbuf);
+    (*env)->GetByteArrayRegion(env, dst, dOff, len, dbuf);
+    xor_op(sbuf, dbuf, len);
+    (*env)->SetByteArrayRegion(env, dst, dOff, len, dbuf);
+
+    free(dbuf);
+    free(sbuf);
+}
+
+JNIEXPORT void JNICALL Java_org_openjdk_bench_java_lang_foreign_xor_GetArrayUnsafeXorOpImpl_xorOp
+  (JNIEnv *env, jobject obj, jlong src, jlong dst, jint len) {
+    jbyte *sbuf = (jbyte*)(void*)src;
+    jbyte *dbuf = (jbyte*)(void*)dst;
+    xor_op(sbuf, dbuf, len);
+}