From 9c98270737cd2019f230e9359bb9298f8df2ca35 Mon Sep 17 00:00:00 2001 From: Jorn Vernee Date: Tue, 14 Nov 2023 11:19:30 +0000 Subject: [PATCH] 8254693: Add Panama feature to pass heap segments to native code Reviewed-by: mcimadamore, lucy, vlivanov --- .../cpu/aarch64/downcallLinker_aarch64.cpp | 153 ++++++++------- .../cpu/aarch64/foreignGlobals_aarch64.cpp | 10 +- .../cpu/aarch64/upcallLinker_aarch64.cpp | 44 +++-- src/hotspot/cpu/arm/downcallLinker_arm.cpp | 5 + src/hotspot/cpu/arm/foreignGlobals_arm.cpp | 2 +- src/hotspot/cpu/arm/upcallLinker_arm.cpp | 1 - src/hotspot/cpu/ppc/downcallLinker_ppc.cpp | 138 ++++++------- src/hotspot/cpu/ppc/foreignGlobals_ppc.cpp | 10 +- src/hotspot/cpu/ppc/upcallLinker_ppc.cpp | 35 ++-- .../cpu/riscv/downcallLinker_riscv.cpp | 150 +++++++------- .../cpu/riscv/foreignGlobals_riscv.cpp | 10 +- src/hotspot/cpu/riscv/upcallLinker_riscv.cpp | 36 ++-- src/hotspot/cpu/s390/downcallLinker_s390.cpp | 139 ++++++------- src/hotspot/cpu/s390/foreignGlobals_s390.cpp | 10 +- src/hotspot/cpu/s390/upcallLinker_s390.cpp | 39 ++-- src/hotspot/cpu/x86/downcallLinker_x86_32.cpp | 5 + src/hotspot/cpu/x86/downcallLinker_x86_64.cpp | 184 ++++++++---------- src/hotspot/cpu/x86/foreignGlobals_x86_32.cpp | 2 +- src/hotspot/cpu/x86/foreignGlobals_x86_64.cpp | 10 +- src/hotspot/cpu/x86/upcallLinker_x86_32.cpp | 1 - src/hotspot/cpu/x86/upcallLinker_x86_64.cpp | 44 +++-- src/hotspot/cpu/zero/downcallLinker_zero.cpp | 5 + src/hotspot/cpu/zero/foreignGlobals_zero.cpp | 2 +- src/hotspot/cpu/zero/upcallLinker_zero.cpp | 3 +- src/hotspot/share/prims/downcallLinker.cpp | 20 +- src/hotspot/share/prims/downcallLinker.hpp | 55 ++++++ src/hotspot/share/prims/foreignGlobals.cpp | 166 +++++++--------- src/hotspot/share/prims/foreignGlobals.hpp | 77 ++++---- src/hotspot/share/prims/nativeEntryPoint.cpp | 8 +- src/hotspot/share/prims/upcallLinker.cpp | 6 +- src/hotspot/share/prims/upcallLinker.hpp | 1 - .../classes/java/lang/foreign/Linker.java | 27 ++- .../java/lang/invoke/NativeMethodHandle.java | 3 +- .../jdk/internal/foreign/abi/Binding.java | 106 +++++++--- .../foreign/abi/BindingInterpreter.java | 3 +- .../foreign/abi/BindingSpecializer.java | 50 +++-- .../foreign/abi/CallingSequenceBuilder.java | 56 +++--- .../internal/foreign/abi/DowncallLinker.java | 16 +- .../internal/foreign/abi/LinkerOptions.java | 10 +- .../jdk/internal/foreign/abi/SharedUtils.java | 6 +- .../foreign/abi/aarch64/CallArranger.java | 20 +- .../foreign/abi/fallback/FallbackLinker.java | 68 +++---- .../foreign/abi/fallback/LibFallback.java | 12 +- .../foreign/abi/ppc64/CallArranger.java | 21 +- .../linux/LinuxRISCV64CallArranger.java | 22 ++- .../abi/s390/linux/LinuxS390CallArranger.java | 21 +- .../foreign/abi/x64/sysv/CallArranger.java | 22 ++- .../foreign/abi/x64/windows/CallArranger.java | 21 +- .../native/libfallbackLinker/fallbackLinker.c | 33 +++- test/jdk/java/foreign/NativeTestHelper.java | 7 +- test/jdk/java/foreign/TestIllegalLink.java | 4 +- .../java/foreign/critical/TestCritical.java | 182 +++++++++++++++++ .../TestCriticalUpcall.java | 2 +- .../foreign/critical/TestStressAllowHeap.java | 111 +++++++++++ test/jdk/java/foreign/critical/libCritical.c | 101 ++++++++++ .../java/foreign/largestub/TestLargeStub.java | 25 ++- .../java/foreign/trivial/TestCritical.java | 81 -------- .../java/lang/foreign/CallOverheadHelper.java | 4 +- .../java/lang/foreign/CriticalCalls.java | 111 +++++++++++ .../bench/java/lang/foreign/Utils.java | 2 +- .../java/lang/foreign/libCriticalCalls.c} | 29 +-- .../bench/java/lang/foreign/libToCString.c | 2 + .../bench/java/lang/foreign/libToJavaString.c | 2 + .../xor/GetArrayCriticalXorOpImpl.java | 11 ++ .../xor/GetArrayElementsXorOpImpl.java | 11 ++ .../xor/GetArrayForeignXorOpCriticalImpl.java | 30 +++ .../foreign/xor/GetArrayForeignXorOpImpl.java | 37 ++++ .../xor/GetArrayForeignXorOpInitImpl.java | 40 ++++ .../foreign/xor/GetArrayRegionXorOpImpl.java | 11 ++ .../foreign/xor/GetArrayUnsafeXorOpImpl.java | 47 +++++ .../bench/java/lang/foreign/xor/XorOp.java | 7 + .../bench/java/lang/foreign/xor/XorTest.java | 130 +++++++++++++ .../bench/java/lang/foreign/xor/libjnitest.c | 75 +++++++ 73 files changed, 1990 insertions(+), 960 deletions(-) create mode 100644 test/jdk/java/foreign/critical/TestCritical.java rename test/jdk/java/foreign/{trivial => critical}/TestCriticalUpcall.java (97%) create mode 100644 test/jdk/java/foreign/critical/TestStressAllowHeap.java create mode 100644 test/jdk/java/foreign/critical/libCritical.c delete mode 100644 test/jdk/java/foreign/trivial/TestCritical.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/CriticalCalls.java rename test/{jdk/java/foreign/trivial/libCritical.c => micro/org/openjdk/bench/java/lang/foreign/libCriticalCalls.c} (76%) create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayCriticalXorOpImpl.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayElementsXorOpImpl.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpCriticalImpl.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpImpl.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpInitImpl.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayRegionXorOpImpl.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayUnsafeXorOpImpl.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/XorOp.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/XorTest.java create mode 100644 test/micro/org/openjdk/bench/java/lang/foreign/xor/libjnitest.c diff --git a/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp index 7571550d0ba..c5bcc38b5f6 100644 --- a/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp @@ -36,63 +36,6 @@ #define __ _masm-> -class DowncallStubGenerator : public StubCodeGenerator { - BasicType* _signature; - int _num_args; - BasicType _ret_bt; - const ABIDescriptor& _abi; - - const GrowableArray& _input_registers; - const GrowableArray& _output_registers; - - bool _needs_return_buffer; - int _captured_state_mask; - bool _needs_transition; - - int _frame_complete; - int _frame_size_slots; - OopMapSet* _oop_maps; -public: - DowncallStubGenerator(CodeBuffer* buffer, - BasicType* signature, - int num_args, - BasicType ret_bt, - const ABIDescriptor& abi, - const GrowableArray& input_registers, - const GrowableArray& output_registers, - bool needs_return_buffer, - int captured_state_mask, - bool needs_transition) - : StubCodeGenerator(buffer, PrintMethodHandleStubs), - _signature(signature), - _num_args(num_args), - _ret_bt(ret_bt), - _abi(abi), - _input_registers(input_registers), - _output_registers(output_registers), - _needs_return_buffer(needs_return_buffer), - _captured_state_mask(captured_state_mask), - _needs_transition(needs_transition), - _frame_complete(0), - _frame_size_slots(0), - _oop_maps(nullptr) { - } - - void generate(); - - int frame_complete() const { - return _frame_complete; - } - - int framesize() const { - return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt)); - } - - OopMapSet* oop_maps() const { - return _oop_maps; - } -}; - static const int native_invoker_code_base_size = 256; static const int native_invoker_size_per_arg = 8; @@ -108,10 +51,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, int code_size = native_invoker_code_base_size + (num_args * native_invoker_size_per_arg); int locs_size = 1; // must be non-zero CodeBuffer code("nep_invoker_blob", code_size, locs_size); - DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi, - input_registers, output_registers, - needs_return_buffer, captured_state_mask, - needs_transition); + StubGenerator g(&code, signature, num_args, ret_bt, abi, + input_registers, output_registers, + needs_return_buffer, captured_state_mask, + needs_transition); g.generate(); code.log_section_sizes("nep_invoker_blob"); @@ -134,7 +77,39 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, return stub; } -void DowncallStubGenerator::generate() { +static constexpr int RFP_BIAS = 16; // skip old rbp and return address + +void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, VMStorage tmp1, VMStorage tmp2) const { + Register r_tmp1 = as_Register(tmp1); + Register r_tmp2 = as_Register(tmp2); + if (reg_oop.is_reg()) { + assert(reg_oop.type() == StorageType::INTEGER, "expected"); + Register reg_oop_reg = as_Register(reg_oop); + if (reg_offset.is_reg()) { + assert(reg_offset.type() == StorageType::INTEGER, "expected"); + __ add(reg_oop_reg, reg_oop_reg, as_Register(reg_offset)); + } else { + assert(reg_offset.is_stack(), "expected"); + assert(reg_offset.stack_size() == 8, "expected long"); + Address offset_addr(rfp, RFP_BIAS + reg_offset.offset()); + __ ldr (r_tmp1, offset_addr); + __ add(reg_oop_reg, reg_oop_reg, r_tmp1); + } + } else { + assert(reg_oop.is_stack(), "expected"); + assert(reg_oop.stack_size() == 8, "expected long"); + assert(reg_offset.is_stack(), "expected"); + assert(reg_offset.stack_size() == 8, "expected long"); + Address offset_addr(rfp, RFP_BIAS + reg_offset.offset()); + Address oop_addr(rfp, RFP_BIAS + reg_oop.offset()); + __ ldr(r_tmp1, offset_addr); + __ ldr(r_tmp2, oop_addr); + __ add(r_tmp1, r_tmp1, r_tmp2); + __ str(r_tmp1, oop_addr); + } +} + +void DowncallLinker::StubGenerator::generate() { enum layout { rfp_off, rfp_off2, @@ -150,23 +125,16 @@ void DowncallStubGenerator::generate() { Register tmp1 = r9; Register tmp2 = r10; - VMStorage shuffle_reg = as_VMStorage(r19); - JavaCallingConvention in_conv; - NativeCallingConvention out_conv(_input_registers); - ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg); - -#ifndef PRODUCT - LogTarget(Trace, foreign, downcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif + GrowableArray java_regs; + ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs); + bool has_objects = false; + GrowableArray filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature, + _num_args, has_objects); + assert(!(_needs_transition && has_objects), "can not pass objects when doing transition"); int allocated_frame_size = 0; assert(_abi._shadow_space_bytes == 0, "not expecting shadow space on AArch64"); - allocated_frame_size += arg_shuffle.out_arg_bytes(); + allocated_frame_size += ForeignGlobals::compute_out_arg_bytes(_input_registers); bool should_save_return_value = !_needs_return_buffer; RegSpiller out_reg_spiller(_output_registers); @@ -193,6 +161,33 @@ void DowncallStubGenerator::generate() { allocated_frame_size += BytesPerWord; } + // The space we have allocated will look like: + // + // FP-> | | + // |---------------------| = frame_bottom_offset = frame_size + // | (optional) | + // | capture state buf | + // |---------------------| = StubLocations::CAPTURED_STATE_BUFFER + // | (optional) | + // | return buffer | + // |---------------------| = StubLocations::RETURN_BUFFER + // SP-> | out/stack args | or | out_reg_spiller area | + // + // Note how the last chunk can be shared, since the 3 uses occur at different times. + + VMStorage shuffle_reg = as_VMStorage(r19); + GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs); + ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg); + +#ifndef PRODUCT + LogTarget(Trace, foreign, downcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + _frame_size_slots = align_up(framesize + (allocated_frame_size >> LogBytesPerInt), 4); assert(is_even(_frame_size_slots/2), "sp not 16-byte aligned"); @@ -218,8 +213,12 @@ void DowncallStubGenerator::generate() { __ stlrw(tmp1, tmp2); } + if (has_objects) { + add_offsets_to_oops(java_regs, as_VMStorage(tmp1), as_VMStorage(tmp2)); + } + __ block_comment("{ argument shuffle"); - arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes, locs); + arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes); __ block_comment("} argument shuffle"); __ blr(as_Register(locs.get(StubLocations::TARGET_ADDRESS))); diff --git a/src/hotspot/cpu/aarch64/foreignGlobals_aarch64.cpp b/src/hotspot/cpu/aarch64/foreignGlobals_aarch64.cpp index 2692054bcdc..9d1a66f8871 100644 --- a/src/hotspot/cpu/aarch64/foreignGlobals_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/foreignGlobals_aarch64.cpp @@ -200,21 +200,13 @@ static void move_v128(MacroAssembler* masm, int out_stk_bias, } } -void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { +void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { Register tmp_reg = as_Register(tmp); for (int i = 0; i < _moves.length(); i++) { Move move = _moves.at(i); VMStorage from_reg = move.from; VMStorage to_reg = move.to; - // replace any placeholders - if (from_reg.type() == StorageType::PLACEHOLDER) { - from_reg = locs.get(from_reg); - } - if (to_reg.type() == StorageType::PLACEHOLDER) { - to_reg = locs.get(to_reg); - } - switch (from_reg.type()) { case StorageType::INTEGER: assert(from_reg.segment_mask() == REG64_MASK, "only 64-bit register supported"); diff --git a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp index fb2f52facaf..03aa1cd6cd7 100644 --- a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp @@ -118,7 +118,6 @@ static const int upcall_stub_code_base_size = 1024; static const int upcall_stub_size_per_arg = 16; address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, - BasicType* in_sig_bt, int total_in_args, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, @@ -126,26 +125,15 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, ResourceMark rm; const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi); const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv); - int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg); + int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg); CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1); - Register shuffle_reg = r19; - JavaCallingConvention out_conv; - NativeCallingConvention in_conv(call_regs._arg_regs); - ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, as_VMStorage(shuffle_reg)); + GrowableArray unfiltered_out_regs; + int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs); int preserved_bytes = SharedRuntime::out_preserve_stack_slots() * VMRegImpl::stack_slot_size; - int stack_bytes = preserved_bytes + arg_shuffle.out_arg_bytes(); + int stack_bytes = preserved_bytes + out_arg_bytes; int out_arg_area = align_up(stack_bytes , StackAlignmentInBytes); -#ifndef PRODUCT - LogTarget(Trace, foreign, upcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif - // out_arg_area (for stack arguments) doubles as shadow space for native calls. // make sure it is big enough. if (out_arg_area < frame::arg_reg_save_area_bytes) { @@ -153,13 +141,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, } int reg_save_area_size = compute_reg_save_area_size(abi); - RegSpiller arg_spilller(call_regs._arg_regs); + RegSpiller arg_spiller(call_regs._arg_regs); RegSpiller result_spiller(call_regs._ret_regs); int shuffle_area_offset = 0; int res_save_area_offset = shuffle_area_offset + out_arg_area; int arg_save_area_offset = res_save_area_offset + result_spiller.spill_size_bytes(); - int reg_save_area_offset = arg_save_area_offset + arg_spilller.spill_size_bytes(); + int reg_save_area_offset = arg_save_area_offset + arg_spiller.spill_size_bytes(); int frame_data_offset = reg_save_area_offset + reg_save_area_size; int frame_bottom_offset = frame_data_offset + sizeof(UpcallStub::FrameData); @@ -173,6 +161,20 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, locs.set(StubLocations::RETURN_BUFFER, abi._scratch1); } + Register shuffle_reg = r19; + GrowableArray in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs); + GrowableArray filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs); + ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, as_VMStorage(shuffle_reg)); + +#ifndef PRODUCT + LogTarget(Trace, foreign, upcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + int frame_size = frame_bottom_offset; frame_size = align_up(frame_size, StackAlignmentInBytes); @@ -212,7 +214,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, // we have to always spill args since we need to do a call to get the thread // (and maybe attach it). - arg_spilller.generate_spill(_masm, arg_save_area_offset); + arg_spiller.generate_spill(_masm, arg_save_area_offset); preserve_callee_saved_registers(_masm, abi, reg_save_area_offset); __ block_comment("{ on_entry"); @@ -225,12 +227,12 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, __ block_comment("} on_entry"); __ block_comment("{ argument shuffle"); - arg_spilller.generate_fill(_masm, arg_save_area_offset); + arg_spiller.generate_fill(_masm, arg_save_area_offset); if (needs_return_buffer) { assert(ret_buf_offset != -1, "no return buffer allocated"); __ lea(as_Register(locs.get(StubLocations::RETURN_BUFFER)), Address(sp, ret_buf_offset)); } - arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0, locs); + arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0); __ block_comment("} argument shuffle"); __ block_comment("{ receiver "); diff --git a/src/hotspot/cpu/arm/downcallLinker_arm.cpp b/src/hotspot/cpu/arm/downcallLinker_arm.cpp index baee7d7a043..eb15424eb38 100644 --- a/src/hotspot/cpu/arm/downcallLinker_arm.cpp +++ b/src/hotspot/cpu/arm/downcallLinker_arm.cpp @@ -38,3 +38,8 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, Unimplemented(); return nullptr; } + +void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, + VMStorage tmp1, VMStorage tmp2) const { + Unimplemented(); +} diff --git a/src/hotspot/cpu/arm/foreignGlobals_arm.cpp b/src/hotspot/cpu/arm/foreignGlobals_arm.cpp index d3a318536bd..5f5a4eb32e0 100644 --- a/src/hotspot/cpu/arm/foreignGlobals_arm.cpp +++ b/src/hotspot/cpu/arm/foreignGlobals_arm.cpp @@ -51,6 +51,6 @@ void RegSpiller::pd_load_reg(MacroAssembler* masm, int offset, VMStorage reg) { Unimplemented(); } -void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { +void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { Unimplemented(); } diff --git a/src/hotspot/cpu/arm/upcallLinker_arm.cpp b/src/hotspot/cpu/arm/upcallLinker_arm.cpp index 3e1fb04218b..c7645f4a033 100644 --- a/src/hotspot/cpu/arm/upcallLinker_arm.cpp +++ b/src/hotspot/cpu/arm/upcallLinker_arm.cpp @@ -26,7 +26,6 @@ #include "utilities/debug.hpp" address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, - BasicType* in_sig_bt, int total_in_args, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, diff --git a/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp index 329b52b882f..5fb9509926f 100644 --- a/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp +++ b/src/hotspot/cpu/ppc/downcallLinker_ppc.cpp @@ -36,63 +36,6 @@ #define __ _masm-> -class DowncallStubGenerator : public StubCodeGenerator { - BasicType* _signature; - int _num_args; - BasicType _ret_bt; - const ABIDescriptor& _abi; - - const GrowableArray& _input_registers; - const GrowableArray& _output_registers; - - bool _needs_return_buffer; - int _captured_state_mask; - bool _needs_transition; - - int _frame_complete; - int _frame_size_slots; - OopMapSet* _oop_maps; -public: - DowncallStubGenerator(CodeBuffer* buffer, - BasicType* signature, - int num_args, - BasicType ret_bt, - const ABIDescriptor& abi, - const GrowableArray& input_registers, - const GrowableArray& output_registers, - bool needs_return_buffer, - int captured_state_mask, - bool needs_transition) - : StubCodeGenerator(buffer, PrintMethodHandleStubs), - _signature(signature), - _num_args(num_args), - _ret_bt(ret_bt), - _abi(abi), - _input_registers(input_registers), - _output_registers(output_registers), - _needs_return_buffer(needs_return_buffer), - _captured_state_mask(captured_state_mask), - _needs_transition(needs_transition), - _frame_complete(0), - _frame_size_slots(0), - _oop_maps(nullptr) { - } - - void generate(); - - int frame_complete() const { - return _frame_complete; - } - - int framesize() const { - return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt)); - } - - OopMapSet* oop_maps() const { - return _oop_maps; - } -}; - static const int native_invoker_code_base_size = 384; static const int native_invoker_size_per_arg = 8; @@ -108,10 +51,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, int code_size = native_invoker_code_base_size + (num_args * native_invoker_size_per_arg); int locs_size = 1; // must be non-zero CodeBuffer code("nep_invoker_blob", code_size, locs_size); - DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi, - input_registers, output_registers, - needs_return_buffer, captured_state_mask, - needs_transition); + StubGenerator g(&code, signature, num_args, ret_bt, abi, + input_registers, output_registers, + needs_return_buffer, captured_state_mask, + needs_transition); g.generate(); code.log_section_sizes("nep_invoker_blob"); @@ -134,23 +77,40 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, return stub; } -void DowncallStubGenerator::generate() { - Register callerSP = R2, // C/C++ uses R2 as TOC, but we can reuse it here - tmp = R11_scratch1, // same as shuffle_reg - call_target_address = R12_scratch2; // same as _abi._scratch2 (ABIv2 requires this reg!) - VMStorage shuffle_reg = _abi._scratch1; - JavaCallingConvention in_conv; - NativeCallingConvention out_conv(_input_registers); - ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg); +static constexpr int FP_BIAS = frame::jit_out_preserve_size; +static const Register callerSP = R2; // C/C++ uses R2 as TOC, but we can reuse it here -#ifndef PRODUCT - LogTarget(Trace, foreign, downcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); +void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, + VMStorage tmp1, VMStorage tmp2) const { + Register r_tmp1 = as_Register(tmp1); + Register r_tmp2 = as_Register(tmp2); + if (reg_oop.is_reg()) { + assert(reg_oop.type() == StorageType::INTEGER, "expected"); + Register reg_oop_reg = as_Register(reg_oop); + if (reg_offset.is_reg()) { + assert(reg_offset.type() == StorageType::INTEGER, "expected"); + __ add(reg_oop_reg, reg_oop_reg, as_Register(reg_offset)); + } else { + assert(reg_offset.is_stack(), "expected"); + assert(reg_offset.stack_size() == 8, "expected long"); + __ ld(r_tmp1, reg_offset.offset() + FP_BIAS, callerSP); + __ add(reg_oop_reg, reg_oop_reg, r_tmp1); + } + } else { + assert(reg_oop.is_stack(), "expected"); + assert(reg_oop.stack_size() == 8, "expected long"); + assert(reg_offset.is_stack(), "expected"); + assert(reg_offset.stack_size() == 8, "expected long"); + __ ld(r_tmp1, reg_offset.offset() + FP_BIAS, callerSP); + __ ld(r_tmp2, reg_oop.offset() + FP_BIAS, callerSP); + __ add(r_tmp1, r_tmp2, r_tmp1); + __ std(r_tmp1, reg_oop.offset() + FP_BIAS, callerSP); } -#endif +} + +void DowncallLinker::StubGenerator::generate() { + Register tmp = R11_scratch1, // same as shuffle_reg + call_target_address = R12_scratch2; // same as _abi._scratch2 (ABIv2 requires this reg!) // Stack frame size computation: // We use the number of input VMStorage elements because PPC64 requires slots for all arguments @@ -191,6 +151,26 @@ void DowncallStubGenerator::generate() { allocated_frame_size += BytesPerWord; } + GrowableArray java_regs; + ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs); + bool has_objects = false; + GrowableArray filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature, + _num_args, has_objects); + assert(!(_needs_transition && has_objects), "can not pass objects when doing transition"); + + GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs); + + ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, _abi._scratch1); + +#ifndef PRODUCT + LogTarget(Trace, foreign, downcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + allocated_frame_size = align_up(allocated_frame_size, StackAlignmentInBytes); _frame_size_slots = allocated_frame_size >> LogBytesPerInt; @@ -216,8 +196,12 @@ void DowncallStubGenerator::generate() { __ stw(R0, in_bytes(JavaThread::thread_state_offset()), R16_thread); } + if (has_objects) { + add_offsets_to_oops(java_regs, _abi._scratch1, _abi._scratch2); + } + __ block_comment("{ argument shuffle"); - arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::jit_out_preserve_size, frame::native_abi_minframe_size, locs); + arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::jit_out_preserve_size, frame::native_abi_minframe_size); __ block_comment("} argument shuffle"); __ call_c(call_target_address); diff --git a/src/hotspot/cpu/ppc/foreignGlobals_ppc.cpp b/src/hotspot/cpu/ppc/foreignGlobals_ppc.cpp index 6c7269bb937..0a37552cd98 100644 --- a/src/hotspot/cpu/ppc/foreignGlobals_ppc.cpp +++ b/src/hotspot/cpu/ppc/foreignGlobals_ppc.cpp @@ -226,21 +226,13 @@ static void move_stack(MacroAssembler* masm, Register callerSP, int in_stk_bias, } } -void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { +void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { Register callerSP = as_Register(tmp); // preset for (int i = 0; i < _moves.length(); i++) { Move move = _moves.at(i); VMStorage from_reg = move.from; VMStorage to_reg = move.to; - // replace any placeholders - if (from_reg.type() == StorageType::PLACEHOLDER) { - from_reg = locs.get(from_reg); - } - if (to_reg.type() == StorageType::PLACEHOLDER) { - to_reg = locs.get(to_reg); - } - switch (from_reg.type()) { case StorageType::INTEGER: move_reg64(masm, out_stk_bias, from_reg, to_reg); diff --git a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp index aaa9952656c..727ebe2b568 100644 --- a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp +++ b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp @@ -119,7 +119,6 @@ static const int upcall_stub_code_base_size = 1024; static const int upcall_stub_size_per_arg = 16; // arg save & restore + move address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, - BasicType* in_sig_bt, int total_in_args, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, @@ -127,27 +126,16 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, ResourceMark rm; const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi); const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv); - int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg); + int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg); CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1); Register callerSP = R2, // C/C++ uses R2 as TOC, but we can reuse it here tmp = R11_scratch1, // same as shuffle_reg call_target_address = R12_scratch2; // same as _abi._scratch2 - VMStorage shuffle_reg = abi._scratch1; - JavaCallingConvention out_conv; - NativeCallingConvention in_conv(call_regs._arg_regs); - ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, shuffle_reg); + GrowableArray unfiltered_out_regs; + int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs); // The Java call uses the JIT ABI, but we also call C. - int out_arg_area = MAX2(frame::jit_out_preserve_size + arg_shuffle.out_arg_bytes(), (int)frame::native_abi_reg_args_size); - -#ifndef PRODUCT - LogTarget(Trace, foreign, upcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif + int out_arg_area = MAX2(frame::jit_out_preserve_size + out_arg_bytes, (int)frame::native_abi_reg_args_size); int reg_save_area_size = compute_reg_save_area_size(abi); RegSpiller arg_spiller(call_regs._arg_regs); @@ -169,6 +157,19 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, locs.set(StubLocations::RETURN_BUFFER, abi._scratch2); } + GrowableArray in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs); + GrowableArray filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs); + ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, abi._scratch1); + +#ifndef PRODUCT + LogTarget(Trace, foreign, upcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + int frame_size = align_up(frame_bottom_offset, StackAlignmentInBytes); // The space we have allocated will look like: @@ -229,7 +230,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, __ addi(as_Register(locs.get(StubLocations::RETURN_BUFFER)), R1_SP, ret_buf_offset); } __ ld(callerSP, _abi0(callers_sp), R1_SP); // preset (used to access caller frame argument slots) - arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::native_abi_minframe_size, frame::jit_out_preserve_size, locs); + arg_shuffle.generate(_masm, as_VMStorage(callerSP), frame::native_abi_minframe_size, frame::jit_out_preserve_size); __ block_comment("} argument shuffle"); __ block_comment("{ receiver "); diff --git a/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp index 67ab410c39b..fefb40f4b1e 100644 --- a/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp +++ b/src/hotspot/cpu/riscv/downcallLinker_riscv.cpp @@ -37,63 +37,6 @@ #define __ _masm-> -class DowncallStubGenerator : public StubCodeGenerator { - BasicType* _signature; - int _num_args; - BasicType _ret_bt; - - const ABIDescriptor& _abi; - const GrowableArray& _input_registers; - const GrowableArray& _output_registers; - - bool _needs_return_buffer; - int _captured_state_mask; - bool _needs_transition; - - int _frame_complete; - int _frame_size_slots; - OopMapSet* _oop_maps; -public: - DowncallStubGenerator(CodeBuffer* buffer, - BasicType* signature, - int num_args, - BasicType ret_bt, - const ABIDescriptor& abi, - const GrowableArray& input_registers, - const GrowableArray& output_registers, - bool needs_return_buffer, - int captured_state_mask, - bool needs_transition) - : StubCodeGenerator(buffer, PrintMethodHandleStubs), - _signature(signature), - _num_args(num_args), - _ret_bt(ret_bt), - _abi(abi), - _input_registers(input_registers), - _output_registers(output_registers), - _needs_return_buffer(needs_return_buffer), - _captured_state_mask(captured_state_mask), - _needs_transition(needs_transition), - _frame_complete(0), - _frame_size_slots(0), - _oop_maps(nullptr) { - } - - void generate(); - - int frame_complete() const { - return _frame_complete; - } - - int framesize() const { - return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt)); - } - - OopMapSet* oop_maps() const { - return _oop_maps; - } -}; - static const int native_invoker_code_base_size = 256; static const int native_invoker_size_per_arg = 8; @@ -109,10 +52,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, int code_size = native_invoker_code_base_size + (num_args * native_invoker_size_per_arg); int locs_size = 1; // must be non-zero CodeBuffer code("nep_invoker_blob", code_size, locs_size); - DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi, - input_registers, output_registers, - needs_return_buffer, captured_state_mask, - needs_transition); + StubGenerator g(&code, signature, num_args, ret_bt, abi, + input_registers, output_registers, + needs_return_buffer, captured_state_mask, + needs_transition); g.generate(); code.log_section_sizes("nep_invoker_blob"); @@ -135,7 +78,36 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, return stub; } -void DowncallStubGenerator::generate() { +static constexpr int FP_BIAS = 0; // sender_sp_offset is 0 on RISCV + +void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, VMStorage tmp1, VMStorage tmp2) const { + Register r_tmp1 = as_Register(tmp1); + Register r_tmp2 = as_Register(tmp2); + if (reg_oop.is_reg()) { + assert(reg_oop.type() == StorageType::INTEGER, "expected"); + Register reg_oop_reg = as_Register(reg_oop); + if (reg_offset.is_reg()) { + assert(reg_offset.type() == StorageType::INTEGER, "expected"); + __ add(reg_oop_reg, reg_oop_reg, as_Register(reg_offset)); + } else { + assert(reg_offset.is_stack(), "expected"); + assert(reg_offset.stack_size() == 8, "expected long"); + __ ld(r_tmp1, Address(fp, FP_BIAS + reg_offset.offset())); + __ add(reg_oop_reg, reg_oop_reg, r_tmp1); + } + } else { + assert(reg_oop.is_stack(), "expected"); + assert(reg_oop.stack_size() == 8, "expected long"); + assert(reg_offset.is_stack(), "expected"); + assert(reg_offset.stack_size() == 8, "expected long"); + __ ld(r_tmp1, Address(fp, FP_BIAS + reg_offset.offset())); + __ ld(r_tmp2, Address(fp, FP_BIAS + reg_oop.offset())); + __ add(r_tmp1, r_tmp2, r_tmp1); + __ sd(r_tmp1, Address(fp, FP_BIAS + reg_oop.offset())); + } +} + +void DowncallLinker::StubGenerator::generate() { enum layout { fp_off, fp_off2, @@ -147,23 +119,16 @@ void DowncallStubGenerator::generate() { // out arg area (e.g. for stack args) }; - VMStorage shuffle_reg = as_VMStorage(x9); - JavaCallingConvention in_conv; - NativeCallingConvention out_conv(_input_registers); - ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg); - -#ifndef PRODUCT - LogTarget(Trace, foreign, downcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif + GrowableArray java_regs; + ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs); + bool has_objects = false; + GrowableArray filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature, + _num_args, has_objects); + assert(!(_needs_transition && has_objects), "can not pass objects when doing transition"); int allocated_frame_size = 0; assert(_abi._shadow_space_bytes == 0, "not expecting shadow space on RISCV64"); - allocated_frame_size += arg_shuffle.out_arg_bytes(); + allocated_frame_size += ForeignGlobals::compute_out_arg_bytes(_input_registers); bool should_save_return_value = !_needs_return_buffer; RegSpiller out_reg_spiller(_output_registers); @@ -190,6 +155,33 @@ void DowncallStubGenerator::generate() { allocated_frame_size += BytesPerWord; } + // The space we have allocated will look like: + // + // FP-> | | + // |---------------------| = frame_bottom_offset = frame_size + // | (optional) | + // | capture state buf | + // |---------------------| = StubLocations::CAPTURED_STATE_BUFFER + // | (optional) | + // | return buffer | + // |---------------------| = StubLocations::RETURN_BUFFER + // SP-> | out/stack args | or | out_reg_spiller area | + // + // Note how the last chunk can be shared, since the 3 uses occur at different times. + + VMStorage shuffle_reg = as_VMStorage(x9); + GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs); + ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg); + +#ifndef PRODUCT + LogTarget(Trace, foreign, downcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + allocated_frame_size = align_up(allocated_frame_size, 16); // _frame_size_slots is in 32-bit stack slots: _frame_size_slots += framesize + (allocated_frame_size >> LogBytesPerInt); @@ -219,8 +211,12 @@ void DowncallStubGenerator::generate() { __ block_comment("} thread java2native"); } + if (has_objects) { + add_offsets_to_oops(java_regs, as_VMStorage(t0), as_VMStorage(t1)); + } + __ block_comment("{ argument shuffle"); - arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes, locs); + arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes); __ block_comment("} argument shuffle"); __ jalr(as_Register(locs.get(StubLocations::TARGET_ADDRESS))); diff --git a/src/hotspot/cpu/riscv/foreignGlobals_riscv.cpp b/src/hotspot/cpu/riscv/foreignGlobals_riscv.cpp index 7ecfe0c38d3..2975b088b0c 100644 --- a/src/hotspot/cpu/riscv/foreignGlobals_riscv.cpp +++ b/src/hotspot/cpu/riscv/foreignGlobals_riscv.cpp @@ -163,21 +163,13 @@ static void move_fp(MacroAssembler* masm, int out_stk_bias, } } -void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { +void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { Register tmp_reg = as_Register(tmp); for (int i = 0; i < _moves.length(); i++) { Move move = _moves.at(i); VMStorage from_reg = move.from; VMStorage to_reg = move.to; - // replace any placeholders - if (from_reg.type() == StorageType::PLACEHOLDER) { - from_reg = locs.get(from_reg); - } - if (to_reg.type() == StorageType::PLACEHOLDER) { - to_reg = locs.get(to_reg); - } - switch (from_reg.type()) { case StorageType::INTEGER: assert(from_reg.segment_mask() == REG64_MASK, "only 64-bit integer register supported"); diff --git a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp index de90694945b..abfea10fd02 100644 --- a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp +++ b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp @@ -118,7 +118,6 @@ static const int upcall_stub_code_base_size = 1024; static const int upcall_stub_size_per_arg = 16; address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, - BasicType* in_sig_bt, int total_in_args, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, @@ -127,26 +126,15 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, ResourceMark rm; const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi); const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv); - int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg); + int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg); CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1); - Register shuffle_reg = x9; - JavaCallingConvention out_conv; - NativeCallingConvention in_conv(call_regs._arg_regs); - ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, as_VMStorage(shuffle_reg)); + GrowableArray unfiltered_out_regs; + int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs); int preserved_bytes = SharedRuntime::out_preserve_stack_slots() * VMRegImpl::stack_slot_size; - int stack_bytes = preserved_bytes + arg_shuffle.out_arg_bytes(); + int stack_bytes = preserved_bytes + out_arg_bytes; int out_arg_area = align_up(stack_bytes , StackAlignmentInBytes); -#ifndef PRODUCT - LogTarget(Trace, foreign, upcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif - // out_arg_area (for stack arguments) doubles as shadow space for native calls. // make sure it is big enough. if (out_arg_area < frame::arg_reg_save_area_bytes) { @@ -174,6 +162,20 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, locs.set(StubLocations::RETURN_BUFFER, abi._scratch1); } + Register shuffle_reg = x9; + GrowableArray in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs); + GrowableArray filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs); + ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, as_VMStorage(shuffle_reg)); + +#ifndef PRODUCT + LogTarget(Trace, foreign, upcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + int frame_size = frame_bottom_offset; frame_size = align_up(frame_size, StackAlignmentInBytes); @@ -252,7 +254,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, __ la(as_Register(locs.get(StubLocations::RETURN_BUFFER)), Address(sp, ret_buf_offset)); } - arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0, locs); + arg_shuffle.generate(_masm, as_VMStorage(shuffle_reg), abi._shadow_space_bytes, 0); __ block_comment("} argument shuffle"); __ block_comment("{ receiver "); diff --git a/src/hotspot/cpu/s390/downcallLinker_s390.cpp b/src/hotspot/cpu/s390/downcallLinker_s390.cpp index f831da90755..d3e11104e48 100644 --- a/src/hotspot/cpu/s390/downcallLinker_s390.cpp +++ b/src/hotspot/cpu/s390/downcallLinker_s390.cpp @@ -36,61 +36,6 @@ #define __ _masm-> -class DowncallStubGenerator : public StubCodeGenerator { - BasicType* _signature; - int _num_args; - BasicType _ret_bt; - const ABIDescriptor& _abi; - - const GrowableArray& _input_registers; - const GrowableArray& _output_registers; - - bool _needs_return_buffer; - int _captured_state_mask; - bool _needs_transition; - - int _frame_complete; - int _frame_size_slots; - OopMapSet* _oop_maps; - public: - DowncallStubGenerator(CodeBuffer* buffer, - BasicType* signature, - int num_args, - BasicType ret_bt, - const ABIDescriptor& abi, - const GrowableArray& input_registers, - const GrowableArray& output_registers, - bool needs_return_buffer, - int captured_state_mask, - bool needs_transition) - :StubCodeGenerator(buffer, PrintMethodHandleStubs), - _signature(signature), - _num_args(num_args), - _ret_bt(ret_bt), - _abi(abi), - _input_registers(input_registers), - _output_registers(output_registers), - _needs_return_buffer(needs_return_buffer), - _captured_state_mask(captured_state_mask), - _needs_transition(needs_transition), - _frame_complete(0), - _frame_size_slots(0), - _oop_maps(nullptr) { - } - void generate(); - int frame_complete() const { - return _frame_complete; - } - - int framesize() const { - return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt)); - } - - OopMapSet* oop_maps() const { - return _oop_maps; - } -}; - static const int native_invoker_code_base_size = 512; static const int native_invoker_size_per_args = 8; @@ -108,10 +53,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, int locs_size = 1; //must be non zero CodeBuffer code("nep_invoker_blob", code_size, locs_size); - DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi, - input_registers, output_registers, - needs_return_buffer, captured_state_mask, - needs_transition); + StubGenerator g(&code, signature, num_args, ret_bt, abi, + input_registers, output_registers, + needs_return_buffer, captured_state_mask, + needs_transition); g.generate(); code.log_section_sizes("nep_invoker_blob"); @@ -134,28 +79,45 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, return stub; } -void DowncallStubGenerator::generate() { +static constexpr int FP_BIAS = frame::z_jit_out_preserve_size; + +void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, + VMStorage tmp1, VMStorage tmp2) const { + Register r_tmp1 = as_Register(tmp1); + Register r_tmp2 = as_Register(tmp2); + Register callerSP = Z_R11; + if (reg_oop.is_reg()) { + assert(reg_oop.type() == StorageType::INTEGER, "expected"); + Register reg_oop_reg = as_Register(reg_oop); + if (reg_offset.is_reg()) { + assert(reg_offset.type() == StorageType::INTEGER, "expected"); + __ z_agr(reg_oop_reg, as_Register(reg_offset)); + } else { + assert(reg_offset.is_stack(), "expected"); + assert(reg_offset.stack_size() == 8, "expected long"); + Address offset_addr(callerSP, FP_BIAS + reg_offset.offset()); + __ z_ag(reg_oop_reg, offset_addr); + } + } else { + assert(reg_oop.is_stack(), "expected"); + assert(reg_oop.stack_size() == 8, "expected long"); + assert(reg_offset.is_stack(), "expected"); + assert(reg_offset.stack_size() == 8, "expected long"); + Address offset_addr(callerSP, FP_BIAS + reg_offset.offset()); + Address oop_addr(callerSP, FP_BIAS + reg_oop.offset()); + __ mem2reg_opt(r_tmp2, oop_addr, true); + __ z_ag(r_tmp2, offset_addr); + __ reg2mem_opt(r_tmp2, oop_addr, true); + } +} + +void DowncallLinker::StubGenerator::generate() { Register call_target_address = Z_R1_scratch, tmp = Z_R0_scratch; - VMStorage shuffle_reg = _abi._scratch1; - - JavaCallingConvention in_conv; - NativeCallingConvention out_conv(_input_registers); - ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg); - -#ifndef PRODUCT - LogTarget(Trace, foreign, downcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif - assert(_abi._shadow_space_bytes == frame::z_abi_160_size, "expected space according to ABI"); int allocated_frame_size = _abi._shadow_space_bytes; - allocated_frame_size += arg_shuffle.out_arg_bytes(); + allocated_frame_size += ForeignGlobals::compute_out_arg_bytes(_input_registers); assert(!_needs_return_buffer, "unexpected needs_return_buffer"); RegSpiller out_reg_spiller(_output_registers); @@ -172,6 +134,26 @@ void DowncallStubGenerator::generate() { __ block_comment("} _captured_state_mask is set"); } + VMStorage shuffle_reg = _abi._scratch1; + GrowableArray java_regs; + ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs); + bool has_objects = false; + GrowableArray filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature, + _num_args, has_objects); + assert(!(_needs_transition && has_objects), "can not pass objects when doing transition"); + + GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs); + ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, _abi._scratch1); + +#ifndef PRODUCT + LogTarget(Trace, foreign, downcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + allocated_frame_size = align_up(allocated_frame_size, StackAlignmentInBytes); _frame_size_slots = allocated_frame_size >> LogBytesPerInt; @@ -196,8 +178,11 @@ void DowncallStubGenerator::generate() { __ set_thread_state(_thread_in_native); __ block_comment("} thread java2native"); } + if (has_objects) { + add_offsets_to_oops(java_regs, _abi._scratch1, _abi._scratch2); + } __ block_comment("{ argument shuffle"); - arg_shuffle.generate(_masm, shuffle_reg, frame::z_jit_out_preserve_size, _abi._shadow_space_bytes, locs); + arg_shuffle.generate(_masm, shuffle_reg, frame::z_jit_out_preserve_size, _abi._shadow_space_bytes); __ block_comment("} argument shuffle"); __ call(as_Register(locs.get(StubLocations::TARGET_ADDRESS))); diff --git a/src/hotspot/cpu/s390/foreignGlobals_s390.cpp b/src/hotspot/cpu/s390/foreignGlobals_s390.cpp index 9796ab4ffe4..6f4697625cb 100644 --- a/src/hotspot/cpu/s390/foreignGlobals_s390.cpp +++ b/src/hotspot/cpu/s390/foreignGlobals_s390.cpp @@ -200,21 +200,13 @@ static void move_stack(MacroAssembler* masm, Register tmp_reg, int in_stk_bias, } } -void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { +void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { Register tmp_reg = as_Register(tmp); for (int i = 0; i < _moves.length(); i++) { Move move = _moves.at(i); VMStorage from_reg = move.from; VMStorage to_reg = move.to; - // replace any placeholders - if (from_reg.type() == StorageType::PLACEHOLDER) { - from_reg = locs.get(from_reg); - } - if (to_reg.type() == StorageType::PLACEHOLDER) { - to_reg = locs.get(to_reg); - } - switch (from_reg.type()) { case StorageType::INTEGER: move_reg(masm, out_stk_bias, from_reg, to_reg); diff --git a/src/hotspot/cpu/s390/upcallLinker_s390.cpp b/src/hotspot/cpu/s390/upcallLinker_s390.cpp index 884f334c806..10be41d6c28 100644 --- a/src/hotspot/cpu/s390/upcallLinker_s390.cpp +++ b/src/hotspot/cpu/s390/upcallLinker_s390.cpp @@ -117,7 +117,6 @@ static void restore_callee_saved_registers(MacroAssembler* _masm, const ABIDescr static const int upcall_stub_code_base_size = 1024; static const int upcall_stub_size_per_arg = 16; // arg save & restore + move address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, - BasicType* in_sig_bt, int total_in_args, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, @@ -125,28 +124,16 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, ResourceMark rm; const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi); const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv); - int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg); + int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg); CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 0); Register call_target_address = Z_R1_scratch; - VMStorage shuffle_reg = abi._scratch1; - JavaCallingConvention out_conv; - NativeCallingConvention in_conv(call_regs._arg_regs); - ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, shuffle_reg); + GrowableArray unfiltered_out_regs; + int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs); // The Java call uses the JIT ABI, but we also call C. - int out_arg_area = MAX2(frame::z_jit_out_preserve_size + arg_shuffle.out_arg_bytes(), (int)frame::z_abi_160_size); - -#ifndef PRODUCT - LogTarget(Trace, foreign, upcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif - + int out_arg_area = MAX2(frame::z_jit_out_preserve_size + out_arg_bytes, (int)frame::z_abi_160_size); int reg_save_area_size = compute_reg_save_area_size(abi); RegSpiller arg_spiller(call_regs._arg_regs); @@ -158,8 +145,22 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, int frame_data_offset = reg_save_area_offset + reg_save_area_size; int frame_bottom_offset = frame_data_offset + sizeof(UpcallStub::FrameData); - int frame_size = align_up(frame_bottom_offset, StackAlignmentInBytes); StubLocations locs; + VMStorage shuffle_reg = abi._scratch1; + GrowableArray in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs); + GrowableArray filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs); + ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, shuffle_reg); + +#ifndef PRODUCT + LogTarget(Trace, foreign, upcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + + int frame_size = align_up(frame_bottom_offset, StackAlignmentInBytes); // The space we have allocated will look like: // @@ -209,7 +210,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, arg_spiller.generate_fill(_masm, arg_save_area_offset); __ block_comment("{ argument shuffle"); - arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, frame::z_jit_out_preserve_size, locs); + arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, frame::z_jit_out_preserve_size); __ block_comment("} argument shuffle"); __ block_comment("{ receiver "); diff --git a/src/hotspot/cpu/x86/downcallLinker_x86_32.cpp b/src/hotspot/cpu/x86/downcallLinker_x86_32.cpp index aabe49a3002..4e549552e96 100644 --- a/src/hotspot/cpu/x86/downcallLinker_x86_32.cpp +++ b/src/hotspot/cpu/x86/downcallLinker_x86_32.cpp @@ -36,3 +36,8 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, Unimplemented(); return nullptr; } + +void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, + VMStorage tmp1, VMStorage tmp2) const { + Unimplemented(); +} diff --git a/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp index 2da44a4e308..1a40429b7bd 100644 --- a/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp +++ b/src/hotspot/cpu/x86/downcallLinker_x86_64.cpp @@ -30,68 +30,10 @@ #include "prims/downcallLinker.hpp" #include "runtime/globals.hpp" #include "runtime/sharedRuntime.hpp" -#include "runtime/stubCodeGenerator.hpp" #include "utilities/formatBuffer.hpp" #define __ _masm-> -class DowncallStubGenerator : public StubCodeGenerator { - BasicType* _signature; - int _num_args; - BasicType _ret_bt; - - const ABIDescriptor& _abi; - const GrowableArray& _input_registers; - const GrowableArray& _output_registers; - - bool _needs_return_buffer; - int _captured_state_mask; - bool _needs_transition; - - int _frame_complete; - int _frame_size_slots; - OopMapSet* _oop_maps; -public: - DowncallStubGenerator(CodeBuffer* buffer, - BasicType* signature, - int num_args, - BasicType ret_bt, - const ABIDescriptor& abi, - const GrowableArray& input_registers, - const GrowableArray& output_registers, - bool needs_return_buffer, - int captured_state_mask, - bool needs_transition) - : StubCodeGenerator(buffer, PrintMethodHandleStubs), - _signature(signature), - _num_args(num_args), - _ret_bt(ret_bt), - _abi(abi), - _input_registers(input_registers), - _output_registers(output_registers), - _needs_return_buffer(needs_return_buffer), - _captured_state_mask(captured_state_mask), - _needs_transition(needs_transition), - _frame_complete(0), - _frame_size_slots(0), - _oop_maps(nullptr) { - } - - void generate(); - - int frame_complete() const { - return _frame_complete; - } - - int framesize() const { // frame size in 64-bit words - return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt)); - } - - OopMapSet* oop_maps() const { - return _oop_maps; - } -}; - static const int native_invoker_code_base_size = 512; static const int native_invoker_size_per_arg = 8; @@ -107,10 +49,10 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, int code_size = native_invoker_code_base_size + (num_args * native_invoker_size_per_arg); int locs_size = 1; // can not be zero CodeBuffer code("nep_invoker_blob", code_size, locs_size); - DowncallStubGenerator g(&code, signature, num_args, ret_bt, abi, - input_registers, output_registers, - needs_return_buffer, captured_state_mask, - needs_transition); + StubGenerator g(&code, signature, num_args, ret_bt, abi, + input_registers, output_registers, + needs_return_buffer, captured_state_mask, + needs_transition); g.generate(); code.log_section_sizes("nep_invoker_blob"); @@ -133,7 +75,41 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, return stub; } -void DowncallStubGenerator::generate() { +static constexpr int RBP_BIAS = 16; // skip old rbp and return address + +void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, + VMStorage tmp1, VMStorage tmp2) const { + if (reg_oop.is_reg()) { + assert(reg_oop.type() == StorageType::INTEGER, "expected"); + if (reg_offset.is_reg()) { + assert(reg_offset.type() == StorageType::INTEGER, "expected"); + __ addptr(as_Register(reg_oop), as_Register(reg_offset)); + } else { + assert(reg_offset.is_stack(), "expected"); + Address offset_addr(rbp, RBP_BIAS + reg_offset.offset()); + __ addptr(as_Register(reg_oop), offset_addr); + } + } else { + assert(reg_oop.is_stack(), "expected"); + assert(reg_offset.is_stack(), "expected"); + Address offset_addr(rbp, RBP_BIAS + reg_offset.offset()); + Address oop_addr(rbp, RBP_BIAS + reg_oop.offset()); + __ movptr(as_Register(tmp1), offset_addr); + __ addptr(oop_addr, as_Register(tmp1)); + } +} + +static void runtime_call(MacroAssembler* _masm, address target) { + __ vzeroupper(); + __ mov(r12, rsp); // remember sp + __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows + __ andptr(rsp, -16); // align stack as required by ABI + __ call(RuntimeAddress(target)); + __ mov(rsp, r12); // restore sp + __ reinit_heapbase(); +} + +void DowncallLinker::StubGenerator::generate() { enum layout { rbp_off, rbp_off2, @@ -146,24 +122,17 @@ void DowncallStubGenerator::generate() { // out arg area (e.g. for stack args) }; - VMStorage shuffle_reg = as_VMStorage(rbx); - JavaCallingConvention in_conv; - NativeCallingConvention out_conv(_input_registers); - ArgumentShuffle arg_shuffle(_signature, _num_args, _signature, _num_args, &in_conv, &out_conv, shuffle_reg); - -#ifndef PRODUCT - LogTarget(Trace, foreign, downcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif + GrowableArray java_regs; + ForeignGlobals::java_calling_convention(_signature, _num_args, java_regs); + bool has_objects = false; + GrowableArray filtered_java_regs = ForeignGlobals::downcall_filter_offset_regs(java_regs, _signature, + _num_args, has_objects); + assert(!(_needs_transition && has_objects), "can not pass objects when doing transition"); // in bytes int allocated_frame_size = 0; allocated_frame_size += _abi._shadow_space_bytes; - allocated_frame_size += arg_shuffle.out_arg_bytes(); + allocated_frame_size += ForeignGlobals::compute_out_arg_bytes(_input_registers); // when we don't use a return buffer we need to spill the return value around our slow path calls bool should_save_return_value = !_needs_return_buffer; @@ -191,6 +160,35 @@ void DowncallStubGenerator::generate() { allocated_frame_size += BytesPerWord; } + // The space we have allocated will look like: + // + // FP-> | | + // |---------------------| = frame_bottom_offset = frame_size + // | (optional) | + // | capture state buf | + // |---------------------| = StubLocations::CAPTURED_STATE_BUFFER + // | (optional) | + // | return buffer | + // |---------------------| = StubLocations::RETURN_BUFFER + // | out/stack args | | | + // |---------------------| or | ret. val. spill area | + // SP-> | shadow space | | | + // + // Note how the last chunk can be shared, since the 3 uses occur at different times. + + GrowableArray out_regs = ForeignGlobals::replace_place_holders(_input_registers, locs); + VMStorage shuffle_reg = as_VMStorage(rbx); + ArgumentShuffle arg_shuffle(filtered_java_regs, out_regs, shuffle_reg); + +#ifndef PRODUCT + LogTarget(Trace, foreign, downcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + allocated_frame_size = align_up(allocated_frame_size, 16); _frame_size_slots += framesize_base + (allocated_frame_size >> LogBytesPerInt); assert(is_even(_frame_size_slots/2), "sp not 16-byte aligned"); @@ -219,12 +217,16 @@ void DowncallStubGenerator::generate() { __ block_comment("} thread java2native"); } + if (has_objects) { + add_offsets_to_oops(java_regs, as_VMStorage(rscratch1), VMStorage::invalid()); + } + __ block_comment("{ argument shuffle"); - arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes, locs); + arg_shuffle.generate(_masm, shuffle_reg, 0, _abi._shadow_space_bytes); __ block_comment("} argument shuffle"); __ call(as_Register(locs.get(StubLocations::TARGET_ADDRESS))); - // this call is assumed not to have killed r15_thread + assert(!_abi.is_volatile_reg(r15_thread), "Call assumed not to kill r15"); if (_needs_return_buffer) { __ movptr(rscratch1, Address(rsp, locs.data_offset(StubLocations::RETURN_BUFFER))); @@ -247,7 +249,6 @@ void DowncallStubGenerator::generate() { if (_captured_state_mask != 0) { __ block_comment("{ save thread local"); - __ vzeroupper(); if (should_save_return_value) { out_reg_spiller.generate_spill(_masm, spill_rsp_offset); @@ -255,12 +256,7 @@ void DowncallStubGenerator::generate() { __ movptr(c_rarg0, Address(rsp, locs.data_offset(StubLocations::CAPTURED_STATE_BUFFER))); __ movl(c_rarg1, _captured_state_mask); - __ mov(r12, rsp); // remember sp - __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows - __ andptr(rsp, -16); // align stack as required by ABI - __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, DowncallLinker::capture_state))); - __ mov(rsp, r12); // restore sp - __ reinit_heapbase(); + runtime_call(_masm, CAST_FROM_FN_PTR(address, DowncallLinker::capture_state)); if (should_save_return_value) { out_reg_spiller.generate_fill(_masm, spill_rsp_offset); @@ -314,19 +310,13 @@ void DowncallStubGenerator::generate() { if (_needs_transition) { __ block_comment("{ L_safepoint_poll_slow_path"); __ bind(L_safepoint_poll_slow_path); - __ vzeroupper(); if (should_save_return_value) { out_reg_spiller.generate_spill(_masm, spill_rsp_offset); } __ mov(c_rarg0, r15_thread); - __ mov(r12, rsp); // remember sp - __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows - __ andptr(rsp, -16); // align stack as required by ABI - __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans))); - __ mov(rsp, r12); // restore sp - __ reinit_heapbase(); + runtime_call(_masm, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)); if (should_save_return_value) { out_reg_spiller.generate_fill(_masm, spill_rsp_offset); @@ -339,18 +329,12 @@ void DowncallStubGenerator::generate() { __ block_comment("{ L_reguard"); __ bind(L_reguard); - __ vzeroupper(); if (should_save_return_value) { out_reg_spiller.generate_spill(_masm, spill_rsp_offset); } - __ mov(r12, rsp); // remember sp - __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows - __ andptr(rsp, -16); // align stack as required by ABI - __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages))); - __ mov(rsp, r12); // restore sp - __ reinit_heapbase(); + runtime_call(_masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)); if (should_save_return_value) { out_reg_spiller.generate_fill(_masm, spill_rsp_offset); diff --git a/src/hotspot/cpu/x86/foreignGlobals_x86_32.cpp b/src/hotspot/cpu/x86/foreignGlobals_x86_32.cpp index 3752bf577d5..c62021c3263 100644 --- a/src/hotspot/cpu/x86/foreignGlobals_x86_32.cpp +++ b/src/hotspot/cpu/x86/foreignGlobals_x86_32.cpp @@ -50,6 +50,6 @@ void RegSpiller::pd_load_reg(MacroAssembler* masm, int offset, VMStorage reg) { Unimplemented(); } -void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { +void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { Unimplemented(); } diff --git a/src/hotspot/cpu/x86/foreignGlobals_x86_64.cpp b/src/hotspot/cpu/x86/foreignGlobals_x86_64.cpp index 8710d4f79f9..658ff6fecdd 100644 --- a/src/hotspot/cpu/x86/foreignGlobals_x86_64.cpp +++ b/src/hotspot/cpu/x86/foreignGlobals_x86_64.cpp @@ -162,21 +162,13 @@ static void move_xmm(MacroAssembler* masm, int out_stk_bias, } } -void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { +void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { Register tmp_reg = as_Register(tmp); for (int i = 0; i < _moves.length(); i++) { Move move = _moves.at(i); VMStorage from_reg = move.from; VMStorage to_reg = move.to; - // replace any placeholders - if (from_reg.type() == StorageType::PLACEHOLDER) { - from_reg = locs.get(from_reg); - } - if (to_reg.type() == StorageType::PLACEHOLDER) { - to_reg = locs.get(to_reg); - } - switch (from_reg.type()) { case StorageType::INTEGER: assert(from_reg.segment_mask() == REG64_MASK, "only 64-bit register supported"); diff --git a/src/hotspot/cpu/x86/upcallLinker_x86_32.cpp b/src/hotspot/cpu/x86/upcallLinker_x86_32.cpp index cb9702f3449..e5075e180d9 100644 --- a/src/hotspot/cpu/x86/upcallLinker_x86_32.cpp +++ b/src/hotspot/cpu/x86/upcallLinker_x86_32.cpp @@ -25,7 +25,6 @@ #include "prims/upcallLinker.hpp" address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, - BasicType* in_sig_bt, int total_in_args, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, diff --git a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp index 395877f1479..ca170c5013e 100644 --- a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp +++ b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp @@ -169,33 +169,21 @@ static const int upcall_stub_code_base_size = 1024; static const int upcall_stub_size_per_arg = 16; address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, - BasicType* in_sig_bt, int total_in_args, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, bool needs_return_buffer, int ret_buf_size) { const ABIDescriptor abi = ForeignGlobals::parse_abi_descriptor(jabi); const CallRegs call_regs = ForeignGlobals::parse_call_regs(jconv); - int code_size = upcall_stub_code_base_size + (total_in_args * upcall_stub_size_per_arg); + int code_size = upcall_stub_code_base_size + (total_out_args * upcall_stub_size_per_arg); CodeBuffer buffer("upcall_stub", code_size, /* locs_size = */ 1); - VMStorage shuffle_reg = as_VMStorage(rbx); - JavaCallingConvention out_conv; - NativeCallingConvention in_conv(call_regs._arg_regs); - ArgumentShuffle arg_shuffle(in_sig_bt, total_in_args, out_sig_bt, total_out_args, &in_conv, &out_conv, shuffle_reg); + GrowableArray unfiltered_out_regs; + int out_arg_bytes = ForeignGlobals::java_calling_convention(out_sig_bt, total_out_args, unfiltered_out_regs); int preserved_bytes = SharedRuntime::out_preserve_stack_slots() * VMRegImpl::stack_slot_size; - int stack_bytes = preserved_bytes + arg_shuffle.out_arg_bytes(); + int stack_bytes = preserved_bytes + out_arg_bytes; int out_arg_area = align_up(stack_bytes , StackAlignmentInBytes); -#ifndef PRODUCT - LogTarget(Trace, foreign, upcall) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - arg_shuffle.print_on(&ls); - } -#endif - // out_arg_area (for stack arguments) doubles as shadow space for native calls. // make sure it is big enough. if (out_arg_area < frame::arg_reg_save_area_bytes) { @@ -203,13 +191,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, } int reg_save_area_size = compute_reg_save_area_size(abi); - RegSpiller arg_spilller(call_regs._arg_regs); + RegSpiller arg_spiller(call_regs._arg_regs); RegSpiller result_spiller(call_regs._ret_regs); int shuffle_area_offset = 0; int res_save_area_offset = shuffle_area_offset + out_arg_area; int arg_save_area_offset = res_save_area_offset + result_spiller.spill_size_bytes(); - int reg_save_area_offset = arg_save_area_offset + arg_spilller.spill_size_bytes(); + int reg_save_area_offset = arg_save_area_offset + arg_spiller.spill_size_bytes(); int frame_data_offset = reg_save_area_offset + reg_save_area_size; int frame_bottom_offset = frame_data_offset + sizeof(UpcallStub::FrameData); @@ -223,6 +211,20 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, locs.set(StubLocations::RETURN_BUFFER, abi._scratch1); } + VMStorage shuffle_reg = as_VMStorage(rbx); + GrowableArray in_regs = ForeignGlobals::replace_place_holders(call_regs._arg_regs, locs); + GrowableArray filtered_out_regs = ForeignGlobals::upcall_filter_receiver_reg(unfiltered_out_regs); + ArgumentShuffle arg_shuffle(in_regs, filtered_out_regs, shuffle_reg); + +#ifndef PRODUCT + LogTarget(Trace, foreign, upcall) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + arg_shuffle.print_on(&ls); + } +#endif + int frame_size = frame_bottom_offset; frame_size = align_up(frame_size, StackAlignmentInBytes); @@ -265,7 +267,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, // we have to always spill args since we need to do a call to get the thread // (and maybe attach it). - arg_spilller.generate_spill(_masm, arg_save_area_offset); + arg_spiller.generate_spill(_masm, arg_save_area_offset); preserve_callee_saved_registers(_masm, abi, reg_save_area_offset); @@ -280,12 +282,12 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, __ block_comment("} on_entry"); __ block_comment("{ argument shuffle"); - arg_spilller.generate_fill(_masm, arg_save_area_offset); + arg_spiller.generate_fill(_masm, arg_save_area_offset); if (needs_return_buffer) { assert(ret_buf_offset != -1, "no return buffer allocated"); __ lea(as_Register(locs.get(StubLocations::RETURN_BUFFER)), Address(rsp, ret_buf_offset)); } - arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, 0, locs); + arg_shuffle.generate(_masm, shuffle_reg, abi._shadow_space_bytes, 0); __ block_comment("} argument shuffle"); __ block_comment("{ receiver "); diff --git a/src/hotspot/cpu/zero/downcallLinker_zero.cpp b/src/hotspot/cpu/zero/downcallLinker_zero.cpp index aabe49a3002..4e549552e96 100644 --- a/src/hotspot/cpu/zero/downcallLinker_zero.cpp +++ b/src/hotspot/cpu/zero/downcallLinker_zero.cpp @@ -36,3 +36,8 @@ RuntimeStub* DowncallLinker::make_downcall_stub(BasicType* signature, Unimplemented(); return nullptr; } + +void DowncallLinker::StubGenerator::pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, + VMStorage tmp1, VMStorage tmp2) const { + Unimplemented(); +} diff --git a/src/hotspot/cpu/zero/foreignGlobals_zero.cpp b/src/hotspot/cpu/zero/foreignGlobals_zero.cpp index 2cd83af6b6e..d28a4c7c2f1 100644 --- a/src/hotspot/cpu/zero/foreignGlobals_zero.cpp +++ b/src/hotspot/cpu/zero/foreignGlobals_zero.cpp @@ -50,6 +50,6 @@ void RegSpiller::pd_load_reg(MacroAssembler* masm, int offset, VMStorage reg) { Unimplemented(); } -void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { +void ArgumentShuffle::pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { Unimplemented(); } diff --git a/src/hotspot/cpu/zero/upcallLinker_zero.cpp b/src/hotspot/cpu/zero/upcallLinker_zero.cpp index cb9702f3449..6447dac86c9 100644 --- a/src/hotspot/cpu/zero/upcallLinker_zero.cpp +++ b/src/hotspot/cpu/zero/upcallLinker_zero.cpp @@ -24,8 +24,7 @@ #include "precompiled.hpp" #include "prims/upcallLinker.hpp" -address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry, - BasicType* in_sig_bt, int total_in_args, +address UpcallLinker::make_upcall_stub(jobject mh, Method* entry, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, diff --git a/src/hotspot/share/prims/downcallLinker.cpp b/src/hotspot/share/prims/downcallLinker.cpp index b2d5ae2e551..76a239a1017 100644 --- a/src/hotspot/share/prims/downcallLinker.cpp +++ b/src/hotspot/share/prims/downcallLinker.cpp @@ -23,6 +23,8 @@ #include "precompiled.hpp" #include "downcallLinker.hpp" +#include "gc/shared/gcLocker.inline.hpp" +#include "runtime/interfaceSupport.inline.hpp" #include #ifdef _WIN64 @@ -30,7 +32,8 @@ #include #endif -void DowncallLinker::capture_state(int32_t* value_ptr, int captured_state_mask) { +// We call this from _thread_in_native, right after a downcall +JVM_LEAF(void, DowncallLinker::capture_state(int32_t* value_ptr, int captured_state_mask)) // keep in synch with jdk.internal.foreign.abi.PreservableValues enum PreservableValues { NONE = 0, @@ -51,4 +54,19 @@ void DowncallLinker::capture_state(int32_t* value_ptr, int captured_state_mask) if (captured_state_mask & ERRNO) { *value_ptr = errno; } +JVM_END + +void DowncallLinker::StubGenerator::add_offsets_to_oops(GrowableArray& java_regs, VMStorage tmp1, VMStorage tmp2) const { + int reg_idx = 0; + for (int sig_idx = 0; sig_idx < _num_args; sig_idx++) { + if (_signature[sig_idx] == T_OBJECT) { + assert(_signature[sig_idx + 1] == T_LONG, "expected offset after oop"); + VMStorage reg_oop = java_regs.at(reg_idx++); + VMStorage reg_offset = java_regs.at(reg_idx++); + sig_idx++; // skip offset + pd_add_offset_to_oop(reg_oop, reg_offset, tmp1, tmp2); + } else if (_signature[sig_idx] != T_VOID) { + reg_idx++; + } + } } diff --git a/src/hotspot/share/prims/downcallLinker.hpp b/src/hotspot/share/prims/downcallLinker.hpp index 6840a3c7f69..188987efbb5 100644 --- a/src/hotspot/share/prims/downcallLinker.hpp +++ b/src/hotspot/share/prims/downcallLinker.hpp @@ -25,6 +25,7 @@ #define SHARE_VM_PRIMS_DOWNCALLLINKER_HPP #include "prims/foreignGlobals.hpp" +#include "runtime/stubCodeGenerator.hpp" class RuntimeStub; @@ -41,6 +42,60 @@ public: bool needs_transition); static void capture_state(int32_t* value_ptr, int captured_state_mask); + + class StubGenerator : public StubCodeGenerator { + BasicType* _signature; + int _num_args; + BasicType _ret_bt; + + const ABIDescriptor& _abi; + const GrowableArray& _input_registers; + const GrowableArray& _output_registers; + + bool _needs_return_buffer; + int _captured_state_mask; + bool _needs_transition; + + int _frame_complete; + int _frame_size_slots; + OopMapSet* _oop_maps; + public: + StubGenerator(CodeBuffer* buffer, + BasicType* signature, + int num_args, + BasicType ret_bt, + const ABIDescriptor& abi, + const GrowableArray& input_registers, + const GrowableArray& output_registers, + bool needs_return_buffer, + int captured_state_mask, + bool needs_transition) + : StubCodeGenerator(buffer, PrintMethodHandleStubs), + _signature(signature), + _num_args(num_args), + _ret_bt(ret_bt), + _abi(abi), + _input_registers(input_registers), + _output_registers(output_registers), + _needs_return_buffer(needs_return_buffer), + _captured_state_mask(captured_state_mask), + _needs_transition(needs_transition), + _frame_complete(0), + _frame_size_slots(0), + _oop_maps(nullptr) { + } + + void generate(); + + int frame_complete() const { return _frame_complete; } + // frame size in words + int framesize() const { return (_frame_size_slots >> (LogBytesPerWord - LogBytesPerInt)); } + + OopMapSet* oop_maps() const { return _oop_maps; } + + void pd_add_offset_to_oop(VMStorage reg_oop, VMStorage reg_offset, VMStorage tmp1, VMStorage tmp2) const; + void add_offsets_to_oops(GrowableArray& java_regs, VMStorage tmp1, VMStorage tmp2) const; + }; }; #endif // SHARE_VM_PRIMS_DOWNCALLLINKER_HPP diff --git a/src/hotspot/share/prims/foreignGlobals.cpp b/src/hotspot/share/prims/foreignGlobals.cpp index 261be4ead59..f3653d29886 100644 --- a/src/hotspot/share/prims/foreignGlobals.cpp +++ b/src/hotspot/share/prims/foreignGlobals.cpp @@ -126,62 +126,70 @@ void ArgumentShuffle::print_on(outputStream* os) const { to_reg.print_on(os); os->print_cr(""); } - os->print_cr("Stack argument bytes: %d", _out_arg_bytes); os->print_cr("}"); } -int NativeCallingConvention::calling_convention(const BasicType* sig_bt, VMStorage* out_regs, int num_args) const { - int src_pos = 0; +int ForeignGlobals::compute_out_arg_bytes(const GrowableArray& out_regs) { uint32_t max_stack_offset = 0; - for (int i = 0; i < num_args; i++) { - switch (sig_bt[i]) { - case T_BOOLEAN: - case T_CHAR: - case T_BYTE: - case T_SHORT: - case T_INT: - case T_FLOAT: { - VMStorage reg = _input_regs.at(src_pos++); - out_regs[i] = reg; - if (reg.is_stack()) - max_stack_offset = MAX2(max_stack_offset, reg.offset() + reg.stack_size()); - break; - } - case T_LONG: - case T_DOUBLE: { - assert((i + 1) < num_args && sig_bt[i + 1] == T_VOID, "expecting half"); - VMStorage reg = _input_regs.at(src_pos++); - out_regs[i] = reg; - if (reg.is_stack()) - max_stack_offset = MAX2(max_stack_offset, reg.offset() + reg.stack_size()); - break; - } - case T_VOID: // Halves of longs and doubles - assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half"); - out_regs[i] = VMStorage::invalid(); - break; - default: - ShouldNotReachHere(); - break; - } + for (VMStorage reg : out_regs) { + if (reg.is_stack()) + max_stack_offset = MAX2(max_stack_offset, reg.offset() + reg.stack_size()); } return align_up(max_stack_offset, 8); } -int JavaCallingConvention::calling_convention(const BasicType* sig_bt, VMStorage* regs, int num_args) const { +int ForeignGlobals::java_calling_convention(const BasicType* signature, int num_args, GrowableArray& out_regs) { VMRegPair* vm_regs = NEW_RESOURCE_ARRAY(VMRegPair, num_args); - int slots = SharedRuntime::java_calling_convention(sig_bt, vm_regs, num_args); + int slots = SharedRuntime::java_calling_convention(signature, vm_regs, num_args); for (int i = 0; i < num_args; i++) { VMRegPair pair = vm_regs[i]; // note, we ignore second here. Signature should consist of register-size values. So there should be // no need for multi-register pairs. - //assert(!pair.first()->is_valid() || pair.is_single_reg(), "must be: %s"); - regs[i] = as_VMStorage(pair.first(), sig_bt[i]); + if (signature[i] != T_VOID) { + out_regs.push(as_VMStorage(pair.first(), signature[i])); + } } return slots << LogBytesPerInt; } -class ComputeMoveOrder: public StackObj { +GrowableArray ForeignGlobals::replace_place_holders(const GrowableArray& regs, const StubLocations& locs) { + GrowableArray result(regs.length()); + for (VMStorage reg : regs) { + result.push(reg.type() == StorageType::PLACEHOLDER ? locs.get(reg) : reg); + } + return result; +} + +GrowableArray ForeignGlobals::upcall_filter_receiver_reg(const GrowableArray& unfiltered_regs) { + GrowableArray out(unfiltered_regs.length() - 1); + // drop first arg reg + for (int i = 1; i < unfiltered_regs.length(); i++) { + out.push(unfiltered_regs.at(i)); + } + return out; +} + +GrowableArray ForeignGlobals::downcall_filter_offset_regs(const GrowableArray& regs, + BasicType* signature, int num_args, + bool& has_objects) { + GrowableArray result(regs.length()); + int reg_idx = 0; + for (int sig_idx = 0; sig_idx < num_args; sig_idx++) { + if (signature[sig_idx] == T_VOID) { + continue; // ignore upper halves + } + + result.push(regs.at(reg_idx++)); + if (signature[sig_idx] == T_OBJECT) { + has_objects = true; + sig_idx++; // skip offset + reg_idx++; + } + } + return result; +} + +class ArgumentShuffle::ComputeMoveOrder: public StackObj { class MoveOperation; // segment_mask_or_size is not taken into account since @@ -258,42 +266,29 @@ class ComputeMoveOrder: public StackObj { }; private: - int _total_in_args; - const VMStorage* _in_regs; - int _total_out_args; - const VMStorage* _out_regs; - const BasicType* _in_sig_bt; + const GrowableArray& _in_regs; + const GrowableArray& _out_regs; VMStorage _tmp_vmreg; GrowableArray _edges; GrowableArray _moves; public: - ComputeMoveOrder(int total_in_args, const VMStorage* in_regs, int total_out_args, VMStorage* out_regs, - const BasicType* in_sig_bt, VMStorage tmp_vmreg) : - _total_in_args(total_in_args), + ComputeMoveOrder(const GrowableArray& in_regs, + const GrowableArray& out_regs, + VMStorage tmp_vmreg) : _in_regs(in_regs), - _total_out_args(total_out_args), _out_regs(out_regs), - _in_sig_bt(in_sig_bt), _tmp_vmreg(tmp_vmreg), - _edges(total_in_args), - _moves(total_in_args) { + _edges(in_regs.length()), + _moves(in_regs.length()) { + assert(in_regs.length() == out_regs.length(), + "stray registers? %d != %d", in_regs.length(), out_regs.length()); } void compute() { - assert(_total_out_args >= _total_in_args, "can only add prefix args"); - // Note that total_out_args args can be greater than total_in_args in the case of upcalls. - // There will be a leading MH receiver arg in the out args in that case. - // - // Leading args in the out args will be ignored below because we iterate from the end of - // the register arrays until !(in_idx >= 0), and total_in_args is smaller. - // - // Stub code adds a move for the receiver to j_rarg0 (and potential other prefix args) manually. - for (int in_idx = _total_in_args - 1, out_idx = _total_out_args - 1; in_idx >= 0; in_idx--, out_idx--) { - BasicType bt = _in_sig_bt[in_idx]; - assert(bt != T_ARRAY, "array not expected"); - VMStorage in_reg = _in_regs[in_idx]; - VMStorage out_reg = _out_regs[out_idx]; + for (int i = 0; i < _in_regs.length(); i++) { + VMStorage in_reg = _in_regs.at(i); + VMStorage out_reg = _out_regs.at(i); if (out_reg.is_stack() || out_reg.is_frame_data()) { // Move operations where the dest is the stack can all be @@ -301,12 +296,8 @@ class ComputeMoveOrder: public StackObj { // The input and output stack spaces are distinct from each other. Move move{in_reg, out_reg}; _moves.push(move); - } else if (in_reg == out_reg - || bt == T_VOID) { - // 1. Can skip non-stack identity moves. - // - // 2. Upper half of long or double (T_VOID). - // Don't need to do anything. + } else if (in_reg == out_reg) { + // Can skip non-stack identity moves. continue; } else { _edges.append(new MoveOperation(in_reg, out_reg)); @@ -314,12 +305,12 @@ class ComputeMoveOrder: public StackObj { } // Break any cycles in the register moves and emit the in the // proper order. - compute_store_order(_tmp_vmreg); + compute_store_order(); } // Walk the edges breaking cycles between moves. The result list // can be walked in order to produce the proper set of loads - void compute_store_order(VMStorage temp_register) { + void compute_store_order() { // Record which moves kill which registers KillerTable killer; // a map of VMStorage -> MoveOperation* for (int i = 0; i < _edges.length(); i++) { @@ -328,7 +319,7 @@ class ComputeMoveOrder: public StackObj { "multiple moves with the same register as destination"); killer.put(s->dst(), s); } - assert(!killer.contains(temp_register), + assert(!killer.contains(_tmp_vmreg), "make sure temp isn't in the registers that are killed"); // create links between loads and stores @@ -350,7 +341,7 @@ class ComputeMoveOrder: public StackObj { start = start->prev(); } if (start->prev() == s) { - start->break_cycle(temp_register); + start->break_cycle(_tmp_vmreg); } // walk the chain forward inserting to store list while (start != nullptr) { @@ -364,31 +355,18 @@ class ComputeMoveOrder: public StackObj { } public: - static GrowableArray compute_move_order(int total_in_args, const VMStorage* in_regs, - int total_out_args, VMStorage* out_regs, - const BasicType* in_sig_bt, VMStorage tmp_vmreg) { - ComputeMoveOrder cmo(total_in_args, in_regs, total_out_args, out_regs, in_sig_bt, tmp_vmreg); + static GrowableArray compute_move_order(const GrowableArray& in_regs, + const GrowableArray& out_regs, + VMStorage tmp_vmreg) { + ComputeMoveOrder cmo(in_regs, out_regs, tmp_vmreg); cmo.compute(); return cmo._moves; } }; ArgumentShuffle::ArgumentShuffle( - BasicType* in_sig_bt, - int num_in_args, - BasicType* out_sig_bt, - int num_out_args, - const CallingConventionClosure* input_conv, - const CallingConventionClosure* output_conv, + const GrowableArray& in_regs, + const GrowableArray& out_regs, VMStorage shuffle_temp) { - - VMStorage* in_regs = NEW_RESOURCE_ARRAY(VMStorage, num_in_args); - input_conv->calling_convention(in_sig_bt, in_regs, num_in_args); - - VMStorage* out_regs = NEW_RESOURCE_ARRAY(VMStorage, num_out_args); - _out_arg_bytes = output_conv->calling_convention(out_sig_bt, out_regs, num_out_args); - - _moves = ComputeMoveOrder::compute_move_order(num_in_args, in_regs, - num_out_args, out_regs, - in_sig_bt, shuffle_temp); + _moves = ComputeMoveOrder::compute_move_order(in_regs, out_regs, shuffle_temp); } diff --git a/src/hotspot/share/prims/foreignGlobals.hpp b/src/hotspot/share/prims/foreignGlobals.hpp index 009fd974d0b..b94da0efe61 100644 --- a/src/hotspot/share/prims/foreignGlobals.hpp +++ b/src/hotspot/share/prims/foreignGlobals.hpp @@ -33,9 +33,8 @@ #include CPU_HEADER(foreignGlobals) -// needs to match StubLocations in Java code. -// placeholder locations to be filled in by -// the code gen code +// Needs to match jdk.internal.foreign.abi.StubLocations in Java code. +// Placeholder locations to be filled in by the code gen code. class StubLocations { public: enum Location : uint32_t { @@ -56,11 +55,7 @@ public: int data_offset(uint32_t loc) const; }; -class CallingConventionClosure { -public: - virtual int calling_convention(const BasicType* sig_bt, VMStorage* regs, int num_args) const = 0; -}; - +// C++ 'mirror' of jdk.internal.foreign.abi.UpcallLinker.CallRegs struct CallRegs { GrowableArray _arg_regs; GrowableArray _ret_regs; @@ -78,25 +73,32 @@ private: public: static bool is_foreign_linker_supported(); + // Helpers for translating from the Java to C++ representation static const ABIDescriptor parse_abi_descriptor(jobject jabi); static const CallRegs parse_call_regs(jobject jconv); static VMStorage parse_vmstorage(oop storage); + + // Adapter from SharedRuntime::java_calling_convention to a 'single VMStorage per value' form. + // Doesn't assign (invalid) storage for T_VOID entries in the signature, which are instead ignored. + static int java_calling_convention(const BasicType* signature, int num_args, GrowableArray& out_regs); + + // Computes the space (in bytes) that is taken up by stack arguments + static int compute_out_arg_bytes(const GrowableArray& out_regs); + + // Replace placeholders (see class StubLocations above) with actual locations in a stub frame + static GrowableArray replace_place_holders(const GrowableArray& regs, const StubLocations& locs); + + // The receiver method handle for upcalls is injected manually into the argument list by the upcall stub. We need a + // filtered list to generate an argument shuffle for the rest of the arguments. + static GrowableArray upcall_filter_receiver_reg(const GrowableArray& unfiltered_regs); + + // Oop offsets are not passed on to native code. + // Filter out the registers of oop offsets to create a list that we can pass to ArgumentShuffle. + static GrowableArray downcall_filter_offset_regs(const GrowableArray& regs, BasicType* signature, + int num_args, bool& has_objects); }; -class JavaCallingConvention : public CallingConventionClosure { -public: - int calling_convention(const BasicType* sig_bt, VMStorage* regs, int num_args) const override; -}; - -class NativeCallingConvention : public CallingConventionClosure { - GrowableArray _input_regs; -public: - NativeCallingConvention(const GrowableArray& input_regs) - : _input_regs(input_regs) {} - - int calling_convention(const BasicType* sig_bt, VMStorage* out_regs, int num_args) const override; -}; - +// Helper class useful for generating spills and fills of a set of registers. class RegSpiller { GrowableArray _regs; int _spill_size_bytes; @@ -117,30 +119,35 @@ private: static void pd_load_reg(MacroAssembler* masm, int offset, VMStorage reg); }; -struct Move { - VMStorage from; - VMStorage to; -}; - +// Class used to compute and generate a shuffle between 2 lists of VMStorages. +// The lists must have the same size. +// Each VMStorage in the source list (in_regs) is shuffled into the +// the VMStorage at the same index in the destination list (out_regs). +// This class helps to automatically compute an order of moves that makes +// sure not to destroy values accidentally by interfering moves, in case the +// source and destination registers overlap. class ArgumentShuffle { private: + class ComputeMoveOrder; + struct Move { + VMStorage from; + VMStorage to; + }; + GrowableArray _moves; - int _out_arg_bytes; public: ArgumentShuffle( - BasicType* in_sig_bt, int num_in_args, - BasicType* out_sig_bt, int num_out_args, - const CallingConventionClosure* input_conv, const CallingConventionClosure* output_conv, + const GrowableArray& in_regs, + const GrowableArray& out_regs, VMStorage shuffle_temp); - int out_arg_bytes() const { return _out_arg_bytes; } - void generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const { - pd_generate(masm, tmp, in_stk_bias, out_stk_bias, locs); + void generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const { + pd_generate(masm, tmp, in_stk_bias, out_stk_bias); } void print_on(outputStream* os) const; private: - void pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias, const StubLocations& locs) const; + void pd_generate(MacroAssembler* masm, VMStorage tmp, int in_stk_bias, int out_stk_bias) const; }; #endif // SHARE_PRIMS_FOREIGN_GLOBALS diff --git a/src/hotspot/share/prims/nativeEntryPoint.cpp b/src/hotspot/share/prims/nativeEntryPoint.cpp index ef962f8ee41..69014c9d64c 100644 --- a/src/hotspot/share/prims/nativeEntryPoint.cpp +++ b/src/hotspot/share/prims/nativeEntryPoint.cpp @@ -52,10 +52,12 @@ JNI_ENTRY(jlong, NEP_makeDowncallStub(JNIEnv* env, jclass _unused, jobject metho GrowableArray input_regs(pcount); for (int i = 0, bt_idx = 0; i < pcount; i++) { oop type_oop = java_lang_invoke_MethodType::ptype(type, i); - assert(java_lang_Class::is_primitive(type_oop), "Only primitives expected"); - BasicType bt = java_lang_Class::primitive_type(type_oop); + BasicType bt = java_lang_Class::as_BasicType(type_oop); basic_type[bt_idx++] = bt; - input_regs.push(ForeignGlobals::parse_vmstorage(arg_moves_oop->obj_at(i))); + oop reg_oop = arg_moves_oop->obj_at(i); + if (reg_oop != nullptr) { + input_regs.push(ForeignGlobals::parse_vmstorage(reg_oop)); + } if (bt == BasicType::T_DOUBLE || bt == BasicType::T_LONG) { basic_type[bt_idx++] = T_VOID; diff --git a/src/hotspot/share/prims/upcallLinker.cpp b/src/hotspot/share/prims/upcallLinker.cpp index acaa3621672..4b924b5302d 100644 --- a/src/hotspot/share/prims/upcallLinker.cpp +++ b/src/hotspot/share/prims/upcallLinker.cpp @@ -178,12 +178,10 @@ JVM_ENTRY(jlong, UL_MakeUpcallStub(JNIEnv *env, jclass unused, jobject mh, jobje assert(i == total_out_args, ""); ret_type = ss.type(); } - // skip receiver - BasicType* in_sig_bt = out_sig_bt + 1; - int total_in_args = total_out_args - 1; return (jlong) UpcallLinker::make_upcall_stub( - mh_j, entry, in_sig_bt, total_in_args, out_sig_bt, total_out_args, ret_type, abi, conv, needs_return_buffer, checked_cast(ret_buf_size)); + mh_j, entry, out_sig_bt, total_out_args, ret_type, + abi, conv, needs_return_buffer, checked_cast(ret_buf_size)); JVM_END #define CC (char*) /*cast a literal from (const char*)*/ diff --git a/src/hotspot/share/prims/upcallLinker.hpp b/src/hotspot/share/prims/upcallLinker.hpp index 529cb7c3911..d80516b2566 100644 --- a/src/hotspot/share/prims/upcallLinker.hpp +++ b/src/hotspot/share/prims/upcallLinker.hpp @@ -38,7 +38,6 @@ private: static void on_exit(UpcallStub::FrameData* context); public: static address make_upcall_stub(jobject mh, Method* entry, - BasicType* in_sig_bt, int total_in_args, BasicType* out_sig_bt, int total_out_args, BasicType ret_type, jobject jabi, jobject jconv, diff --git a/src/java.base/share/classes/java/lang/foreign/Linker.java b/src/java.base/share/classes/java/lang/foreign/Linker.java index 0ca3470c479..6c89a1ec903 100644 --- a/src/java.base/share/classes/java/lang/foreign/Linker.java +++ b/src/java.base/share/classes/java/lang/foreign/Linker.java @@ -669,8 +669,13 @@ public sealed interface Linker permits AbstractLinker { * might attempt to access the contents of the segment. As such, one of the * exceptions specified by the {@link MemorySegment#get(ValueLayout.OfByte, long)} or * the {@link MemorySegment#copy(MemorySegment, long, MemorySegment, long, long)} - * methods may be thrown. The returned method handle will additionally throw - * {@link NullPointerException} if any argument passed to it is {@code null}. + * methods may be thrown. If an argument is a {@link MemorySegment} whose + * corresponding layout is an {@linkplain AddressLayout address layout}, the linker + * will throw an {@link IllegalArgumentException} if the segment is a heap memory + * segment, unless heap memory segments are explicitly allowed through the + * {@link Linker.Option#critical(boolean)} linker option. The returned method handle + * will additionally throw {@link NullPointerException} if any argument passed to it + * is {@code null}. * * @param function the function descriptor of the target foreign function * @param options the linker options associated with this linkage request @@ -914,9 +919,23 @@ public sealed interface Linker permits AbstractLinker { *

* Using this linker option when linking non-critical functions is likely to have * adverse effects, such as loss of performance or JVM crashes. + *

+ * Critical functions can optionally allow access to the Java heap. This allows + * clients to pass heap memory segments as addresses, where normally only off-heap + * memory segments would be allowed. The memory region inside the Java heap is + * exposed through a temporary native address that is valid for the duration of + * the function call. Use of this mechanism is therefore only recommended when a + * function needs to do short-lived access to Java heap memory, and copying the + * relevant data to an off-heap memory segment would be prohibitive in terms of + * performance. + * + * @param allowHeapAccess whether the linked function should allow access to the + * Java heap. */ - static Option critical() { - return LinkerOptions.Critical.INSTANCE; + static Option critical(boolean allowHeapAccess) { + return allowHeapAccess + ? LinkerOptions.Critical.ALLOW_HEAP + : LinkerOptions.Critical.DONT_ALLOW_HEAP; } } } diff --git a/src/java.base/share/classes/java/lang/invoke/NativeMethodHandle.java b/src/java.base/share/classes/java/lang/invoke/NativeMethodHandle.java index f7e5abbc073..4d09b96775d 100644 --- a/src/java.base/share/classes/java/lang/invoke/NativeMethodHandle.java +++ b/src/java.base/share/classes/java/lang/invoke/NativeMethodHandle.java @@ -77,7 +77,8 @@ import static java.lang.invoke.MethodHandleStatics.newInternalError; || pType == int.class || pType == float.class || pType == double.class - || pType == void.class); + || pType == void.class + || pType == Object.class); } private static final MemberName.Factory IMPL_NAMES = MemberName.getFactory(); diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/Binding.java b/src/java.base/share/classes/jdk/internal/foreign/abi/Binding.java index 875b5d998e3..1bba50a78ce 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/Binding.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/Binding.java @@ -24,6 +24,7 @@ */ package jdk.internal.foreign.abi; +import jdk.internal.foreign.AbstractMemorySegmentImpl; import jdk.internal.foreign.Utils; import jdk.internal.foreign.abi.BindingInterpreter.LoadFunc; import jdk.internal.foreign.abi.BindingInterpreter.StoreFunc; @@ -201,7 +202,7 @@ public sealed interface Binding { LoadFunc loadFunc, SegmentAllocator allocator); private static void checkType(Class type) { - if (!type.isPrimitive() || type == void.class) + if (type != Object.class && (!type.isPrimitive() || type == void.class)) throw new IllegalArgumentException("Illegal type: " + type); } @@ -267,8 +268,21 @@ public sealed interface Binding { return new BoxAddress(byteSize, 1, true); } - static UnboxAddress unboxAddress() { - return UnboxAddress.INSTANCE; + // alias + static SegmentOffset unboxAddress() { + return segmentOffsetNoAllowHeap(); + } + + static SegmentBase segmentBase() { + return SegmentBase.INSTANCE; + } + + static SegmentOffset segmentOffsetAllowHeap() { + return SegmentOffset.INSTANCE_ALLOW_HEAP; + } + + static SegmentOffset segmentOffsetNoAllowHeap() { + return SegmentOffset.INSTANCE_NO_ALLOW_HEAP; } static Dup dup() { @@ -413,6 +427,21 @@ public sealed interface Binding { return this; } + public Binding.Builder segmentBase() { + bindings.add(Binding.segmentBase()); + return this; + } + + public Binding.Builder segmentOffsetAllowHeap() { + bindings.add(Binding.segmentOffsetAllowHeap()); + return this; + } + + public Binding.Builder segmentOffsetNoAllowHeap() { + bindings.add(Binding.segmentOffsetNoAllowHeap()); + return this; + } + public Binding.Builder dup() { bindings.add(Binding.dup()); return this; @@ -448,8 +477,10 @@ public sealed interface Binding { /** * VM_STORE([storage location], [type]) - * Pops a [type] from the operand stack, and moves it to [storage location] - * The [type] must be one of byte, short, char, int, long, float, or double + * Pops a [type] from the operand stack, and moves it to [storage location] + * The [type] must be one of byte, short, char, int, long, float, or double. + * [storage location] may be 'null', indicating that this value should be passed + * to the VM stub, but does not have an explicit target register (e.g. oop offsets) */ record VMStore(VMStorage storage, Class type) implements Move { @@ -469,8 +500,8 @@ public sealed interface Binding { /** * VM_LOAD([storage location], [type]) - * Loads a [type] from [storage location], and pushes it onto the operand stack. - * The [type] must be one of byte, short, char, int, long, float, or double + * Loads a [type] from [storage location], and pushes it onto the operand stack. + * The [type] must be one of byte, short, char, int, long, float, or double */ record VMLoad(VMStorage storage, Class type) implements Move { @@ -493,9 +524,9 @@ public sealed interface Binding { /** * BUFFER_STORE([offset into memory region], [type], [width]) - * Pops a [type] from the operand stack, then pops a MemorySegment from the operand stack. - * Stores [width] bytes of the value contained in the [type] to [offset into memory region]. - * The [type] must be one of byte, short, char, int, long, float, or double + * Pops a [type] from the operand stack, then pops a MemorySegment from the operand stack. + * Stores [width] bytes of the value contained in the [type] to [offset into memory region]. + * The [type] must be one of byte, short, char, int, long, float, or double */ record BufferStore(long offset, Class type, int byteWidth) implements Dereference { @@ -550,9 +581,9 @@ public sealed interface Binding { /** * BUFFER_LOAD([offset into memory region], [type], [width]) - * Pops a MemorySegment from the operand stack, - * and then loads [width] bytes from it at [offset into memory region], into a [type]. - * The [type] must be one of byte, short, char, int, long, float, or double + * Pops a MemorySegment from the operand stack, + * and then loads [width] bytes from it at [offset into memory region], into a [type]. + * The [type] must be one of byte, short, char, int, long, float, or double */ record BufferLoad(long offset, Class type, int byteWidth) implements Dereference { @@ -606,8 +637,8 @@ public sealed interface Binding { /** * COPY([size], [alignment]) * Creates a new MemorySegment with the given [size] and [alignment], - * and copies contents from a MemorySegment popped from the top of the operand stack into this new buffer, - * and pushes the new buffer onto the operand stack + * and copies contents from a MemorySegment popped from the top of the operand stack into this new buffer, + * and pushes the new buffer onto the operand stack */ record Copy(long size, long alignment) implements Binding { private static MemorySegment copyBuffer(MemorySegment operand, long size, long alignment, SegmentAllocator allocator) { @@ -653,12 +684,37 @@ public sealed interface Binding { } /** - * UNBOX_ADDRESS() - * Pops a 'MemoryAddress' from the operand stack, converts it to a 'long', - * with the given size, and pushes that onto the operand stack + * SEGMENT_BASE() + * Pops a MemorySegment from the stack, retrieves the heap base object from it, or null if there is none + * (See: AbstractMemorySegmentImpl::unsafeGetBase), and pushes the result onto the operand stack. */ - record UnboxAddress() implements Binding { - static final UnboxAddress INSTANCE = new UnboxAddress(); + record SegmentBase() implements Binding { + static final SegmentBase INSTANCE = new SegmentBase(); + + @Override + public void verify(Deque> stack) { + Class actualType = stack.pop(); + SharedUtils.checkType(actualType, MemorySegment.class); + stack.push(Object.class); + } + + @Override + public void interpret(Deque stack, StoreFunc storeFunc, + LoadFunc loadFunc, SegmentAllocator allocator) { + stack.push(((AbstractMemorySegmentImpl)stack.pop()).unsafeGetBase()); + } + } + + /** + * SEGMENT_OFFSET([allowHeap]) + * Pops a MemorySegment from the stack, retrieves the offset from it, + * (See: AbstractMemorySegmentImpl::unsafeGetOffset), and pushes the result onto the operand stack. + * Note that for heap segments, the offset is a virtual address into the heap base object. + * If [allowHeap] is 'false' an exception will be thrown for heap segments (See SharedUtils::checkNative). + */ + record SegmentOffset(boolean allowHeap) implements Binding { + static final SegmentOffset INSTANCE_NO_ALLOW_HEAP = new SegmentOffset(false); + static final SegmentOffset INSTANCE_ALLOW_HEAP = new SegmentOffset(true); @Override public void verify(Deque> stack) { @@ -670,14 +726,18 @@ public sealed interface Binding { @Override public void interpret(Deque stack, StoreFunc storeFunc, LoadFunc loadFunc, SegmentAllocator allocator) { - stack.push(SharedUtils.unboxSegment((MemorySegment)stack.pop())); + MemorySegment operand = (MemorySegment) stack.pop(); + if (!allowHeap) { + SharedUtils.checkNative(operand); + } + stack.push(((AbstractMemorySegmentImpl)operand).unsafeGetOffset()); } } /** * BOX_ADDRESS() - * Pops a 'long' from the operand stack, converts it to a 'MemorySegment', with the given size and memory scope - * (either the context scope, or the global scope), and pushes that onto the operand stack. + * Pops a 'long' from the operand stack, converts it to a 'MemorySegment', with the given size and memory scope + * (either the context scope, or the global scope), and pushes that onto the operand stack. */ record BoxAddress(long size, long align, boolean needsScope) implements Binding { diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/BindingInterpreter.java b/src/java.base/share/classes/jdk/internal/foreign/abi/BindingInterpreter.java index df6fdaba9a7..9434e70ac41 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/BindingInterpreter.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/BindingInterpreter.java @@ -27,12 +27,13 @@ package jdk.internal.foreign.abi; import java.lang.foreign.SegmentAllocator; import java.util.ArrayDeque; import java.util.Deque; +import java.util.LinkedList; import java.util.List; public class BindingInterpreter { static void unbox(Object arg, List bindings, StoreFunc storeFunc, SegmentAllocator allocator) { - Deque stack = new ArrayDeque<>(); + Deque stack = new LinkedList<>(); // Use LinkedList as a null-friendly Deque for null segment bases stack.push(arg); for (Binding b : bindings) { diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/BindingSpecializer.java b/src/java.base/share/classes/jdk/internal/foreign/abi/BindingSpecializer.java index 3855a49c389..713a2fcfac3 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/BindingSpecializer.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/BindingSpecializer.java @@ -39,9 +39,10 @@ import jdk.internal.foreign.abi.Binding.BufferStore; import jdk.internal.foreign.abi.Binding.Cast; import jdk.internal.foreign.abi.Binding.Copy; import jdk.internal.foreign.abi.Binding.Dup; +import jdk.internal.foreign.abi.Binding.SegmentBase; +import jdk.internal.foreign.abi.Binding.SegmentOffset; import jdk.internal.foreign.abi.Binding.ShiftLeft; import jdk.internal.foreign.abi.Binding.ShiftRight; -import jdk.internal.foreign.abi.Binding.UnboxAddress; import jdk.internal.foreign.abi.Binding.VMLoad; import jdk.internal.foreign.abi.Binding.VMStore; import sun.security.action.GetBooleanAction; @@ -103,7 +104,9 @@ public class BindingSpecializer { private static final MethodTypeDesc MTD_SCOPE = MethodTypeDesc.of(CD_MemorySegment_Scope); private static final MethodTypeDesc MTD_SESSION_IMPL = MethodTypeDesc.of(CD_MemorySessionImpl); private static final MethodTypeDesc MTD_CLOSE = MTD_void; - private static final MethodTypeDesc MTD_UNBOX_SEGMENT = MethodTypeDesc.of(CD_long, CD_MemorySegment); + private static final MethodTypeDesc MTD_CHECK_NATIVE = MethodTypeDesc.of(CD_void, CD_MemorySegment); + private static final MethodTypeDesc MTD_UNSAFE_GET_BASE = MethodTypeDesc.of(CD_Object); + private static final MethodTypeDesc MTD_UNSAFE_GET_OFFSET = MethodTypeDesc.of(CD_long); private static final MethodTypeDesc MTD_COPY = MethodTypeDesc.of(CD_void, CD_MemorySegment, CD_long, CD_MemorySegment, CD_long, CD_long); private static final MethodTypeDesc MTD_LONG_TO_ADDRESS_NO_SCOPE = MethodTypeDesc.of(CD_MemorySegment, CD_long, CD_long, CD_long); private static final MethodTypeDesc MTD_LONG_TO_ADDRESS_SCOPE = MethodTypeDesc.of(CD_MemorySegment, CD_long, CD_long, CD_long, CD_MemorySessionImpl); @@ -456,18 +459,19 @@ public class BindingSpecializer { private void doBindings(List bindings) { for (Binding binding : bindings) { switch (binding) { - case VMStore vmStore -> emitVMStore(vmStore); - case VMLoad vmLoad -> emitVMLoad(vmLoad); - case BufferStore bufferStore -> emitBufferStore(bufferStore); - case BufferLoad bufferLoad -> emitBufferLoad(bufferLoad); - case Copy copy -> emitCopyBuffer(copy); - case Allocate allocate -> emitAllocBuffer(allocate); - case BoxAddress boxAddress -> emitBoxAddress(boxAddress); - case UnboxAddress unused -> emitUnboxAddress(); - case Dup unused -> emitDupBinding(); - case ShiftLeft shiftLeft -> emitShiftLeft(shiftLeft); - case ShiftRight shiftRight -> emitShiftRight(shiftRight); - case Cast cast -> emitCast(cast); + case VMStore vmStore -> emitVMStore(vmStore); + case VMLoad vmLoad -> emitVMLoad(vmLoad); + case BufferStore bufferStore -> emitBufferStore(bufferStore); + case BufferLoad bufferLoad -> emitBufferLoad(bufferLoad); + case Copy copy -> emitCopyBuffer(copy); + case Allocate allocate -> emitAllocBuffer(allocate); + case BoxAddress boxAddress -> emitBoxAddress(boxAddress); + case SegmentBase unused -> emitSegmentBase(); + case SegmentOffset segmentOffset -> emitSegmentOffset(segmentOffset); + case Dup unused -> emitDupBinding(); + case ShiftLeft shiftLeft -> emitShiftLeft(shiftLeft); + case ShiftRight shiftRight -> emitShiftRight(shiftRight); + case Cast cast -> emitCast(cast); } } } @@ -775,9 +779,23 @@ public class BindingSpecializer { pushType(toType); } - private void emitUnboxAddress() { + private void emitSegmentBase() { popType(MemorySegment.class); - cb.invokestatic(CD_SharedUtils, "unboxSegment", MTD_UNBOX_SEGMENT); + cb.checkcast(CD_AbstractMemorySegmentImpl); + cb.invokevirtual(CD_AbstractMemorySegmentImpl, "unsafeGetBase", MTD_UNSAFE_GET_BASE); + pushType(Object.class); + } + + private void emitSegmentOffset(SegmentOffset segmentOffset) { + popType(MemorySegment.class); + + if (!segmentOffset.allowHeap()) { + cb.dup(); + cb.invokestatic(CD_SharedUtils, "checkNative", MTD_CHECK_NATIVE); + } + cb.checkcast(CD_AbstractMemorySegmentImpl); + cb.invokevirtual(CD_AbstractMemorySegmentImpl, "unsafeGetOffset", MTD_UNSAFE_GET_OFFSET); + pushType(long.class); } diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/CallingSequenceBuilder.java b/src/java.base/share/classes/jdk/internal/foreign/abi/CallingSequenceBuilder.java index 2683dc4542f..a439df3eb1d 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/CallingSequenceBuilder.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/CallingSequenceBuilder.java @@ -25,16 +25,17 @@ package jdk.internal.foreign.abi; import jdk.internal.foreign.Utils; -import jdk.internal.foreign.abi.Binding.Allocate; +import jdk.internal.foreign.abi.Binding.*; import jdk.internal.foreign.abi.Binding.BoxAddress; import jdk.internal.foreign.abi.Binding.BufferLoad; import jdk.internal.foreign.abi.Binding.BufferStore; import jdk.internal.foreign.abi.Binding.Cast; import jdk.internal.foreign.abi.Binding.Copy; import jdk.internal.foreign.abi.Binding.Dup; +import jdk.internal.foreign.abi.Binding.SegmentBase; +import jdk.internal.foreign.abi.Binding.SegmentOffset; import jdk.internal.foreign.abi.Binding.ShiftLeft; import jdk.internal.foreign.abi.Binding.ShiftRight; -import jdk.internal.foreign.abi.Binding.UnboxAddress; import jdk.internal.foreign.abi.Binding.VMLoad; import jdk.internal.foreign.abi.Binding.VMStore; import sun.security.action.GetPropertyAction; @@ -217,19 +218,19 @@ public class CallingSequenceBuilder { static boolean isUnbox(Binding binding) { return switch (binding) { - case VMStore unused -> true; - case BufferLoad unused -> true; - case Copy unused -> true; - case UnboxAddress unused -> true; - case Dup unused -> true; - case ShiftLeft unused -> true; - case ShiftRight unused -> true; - case Cast unused -> true; - - case VMLoad unused -> false; - case BufferStore unused -> false; - case Allocate unused -> false; - case BoxAddress unused -> false; + case VMStore unused -> true; + case BufferLoad unused -> true; + case Copy unused -> true; + case Dup unused -> true; + case SegmentBase unused -> true; + case SegmentOffset unused -> true; + case ShiftLeft unused -> true; + case ShiftRight unused -> true; + case Cast unused -> true; + case VMLoad unused -> false; + case BufferStore unused -> false; + case Allocate unused -> false; + case BoxAddress unused -> false; }; } @@ -252,19 +253,20 @@ public class CallingSequenceBuilder { static boolean isBox(Binding binding) { return switch (binding) { - case VMLoad unused -> true; - case BufferStore unused -> true; - case Copy unused -> true; - case Allocate unused -> true; - case BoxAddress unused -> true; - case Dup unused -> true; - case ShiftLeft unused -> true; - case ShiftRight unused -> true; - case Cast unused -> true; + case VMLoad unused -> true; + case BufferStore unused -> true; + case Copy unused -> true; + case Allocate unused -> true; + case BoxAddress unused -> true; + case Dup unused -> true; + case ShiftLeft unused -> true; + case ShiftRight unused -> true; + case Cast unused -> true; - case VMStore unused -> false; - case BufferLoad unused -> false; - case UnboxAddress unused -> false; + case VMStore unused -> false; + case BufferLoad unused -> false; + case SegmentBase unused -> false; + case SegmentOffset unused -> false; }; } diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/DowncallLinker.java b/src/java.base/share/classes/jdk/internal/foreign/abi/DowncallLinker.java index a092d242a53..44fee62cb6f 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/DowncallLinker.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/DowncallLinker.java @@ -98,9 +98,7 @@ public class DowncallLinker { if (USE_SPEC) { handle = BindingSpecializer.specializeDowncall(handle, callingSequence, abi); } else { - Map argIndexMap = SharedUtils.indexMap(argMoves); - - InvocationData invData = new InvocationData(handle, callingSequence, argIndexMap); + InvocationData invData = new InvocationData(handle, callingSequence); handle = insertArguments(MH_INVOKE_INTERP_BINDINGS.bindTo(this), 2, invData); MethodType interpType = callingSequence.callerMethodType(); if (callingSequence.needsReturnBuffer()) { @@ -151,7 +149,7 @@ public class DowncallLinker { return Arrays.stream(moves).map(Binding.Move::storage).toArray(VMStorage[]::new); } - private record InvocationData(MethodHandle leaf, CallingSequence callingSequence, Map argIndexMap) {} + private record InvocationData(MethodHandle leaf, CallingSequence callingSequence) {} Object invokeInterpBindings(SegmentAllocator allocator, Object[] args, InvocationData invData) throws Throwable { Arena unboxArena = callingSequence.allocationSize() != 0 @@ -172,6 +170,13 @@ public class DowncallLinker { } Object[] leafArgs = new Object[invData.leaf.type().parameterCount()]; + BindingInterpreter.StoreFunc storeFunc = new BindingInterpreter.StoreFunc() { + int argOffset = 0; + @Override + public void store(VMStorage storage, Object o) { + leafArgs[argOffset++] = o; + } + }; for (int i = 0; i < args.length; i++) { Object arg = args[i]; if (callingSequence.functionDesc().argumentLayouts().get(i) instanceof AddressLayout) { @@ -183,8 +188,7 @@ public class DowncallLinker { acquiredScopes.add(sessionImpl); } } - BindingInterpreter.unbox(arg, callingSequence.argumentBindings(i), - (storage, value) -> leafArgs[invData.argIndexMap.get(storage)] = value, unboxArena); + BindingInterpreter.unbox(arg, callingSequence.argumentBindings(i), storeFunc, unboxArena); } // call leaf diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/LinkerOptions.java b/src/java.base/share/classes/jdk/internal/foreign/abi/LinkerOptions.java index 3729093f09e..fcc98ecccc0 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/LinkerOptions.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/LinkerOptions.java @@ -106,6 +106,11 @@ public class LinkerOptions { return c != null; } + public boolean allowsHeapAccess() { + Critical c = getOption(Critical.class); + return c != null && c.allowHeapAccess(); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -145,8 +150,9 @@ public class LinkerOptions { } } - public record Critical() implements LinkerOptionImpl { - public static Critical INSTANCE = new Critical(); + public record Critical(boolean allowHeapAccess) implements LinkerOptionImpl { + public static Critical ALLOW_HEAP = new Critical(true); + public static Critical DONT_ALLOW_HEAP = new Critical(false); @Override public void validateForDowncall(FunctionDescriptor descriptor) { diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/SharedUtils.java b/src/java.base/share/classes/jdk/internal/foreign/abi/SharedUtils.java index 07427fca749..5f625242182 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/SharedUtils.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/SharedUtils.java @@ -314,10 +314,14 @@ public final class SharedUtils { } } - public static long unboxSegment(MemorySegment segment) { + public static void checkNative(MemorySegment segment) { if (!segment.isNative()) { throw new IllegalArgumentException("Heap segment not allowed: " + segment); } + } + + public static long unboxSegment(MemorySegment segment) { + checkNative(segment); return segment.address(); } diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/aarch64/CallArranger.java b/src/java.base/share/classes/jdk/internal/foreign/abi/aarch64/CallArranger.java index efd5090e8d9..9eb4910e580 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/aarch64/CallArranger.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/aarch64/CallArranger.java @@ -151,8 +151,8 @@ public abstract class CallArranger { boolean forVariadicFunction = options.isVariadicFunction(); - BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, forVariadicFunction); - BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, forVariadicFunction) : new BoxBindingCalculator(false); + BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, forVariadicFunction, options.allowsHeapAccess()); + BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, forVariadicFunction, false) : new BoxBindingCalculator(false); boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout()); if (returnInMemory) { @@ -388,11 +388,13 @@ public abstract class CallArranger { class UnboxBindingCalculator extends BindingCalculator { protected final boolean forArguments; protected final boolean forVariadicFunction; + private final boolean useAddressPairs; - UnboxBindingCalculator(boolean forArguments, boolean forVariadicFunction) { + UnboxBindingCalculator(boolean forArguments, boolean forVariadicFunction, boolean useAddressPairs) { super(forArguments, forVariadicFunction); this.forArguments = forArguments; this.forVariadicFunction = forVariadicFunction; + this.useAddressPairs = useAddressPairs; } @Override @@ -432,9 +434,17 @@ public abstract class CallArranger { bindings.vmStore(storage, long.class); } case POINTER -> { - bindings.unboxAddress(); VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER, (ValueLayout) layout); - bindings.vmStore(storage, long.class); + if (useAddressPairs) { + bindings.dup() + .segmentBase() + .vmStore(storage, Object.class) + .segmentOffsetAllowHeap() + .vmStore(null, long.class); + } else { + bindings.unboxAddress(); + bindings.vmStore(storage, long.class); + } } case INTEGER -> { VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER, (ValueLayout) layout); diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/fallback/FallbackLinker.java b/src/java.base/share/classes/jdk/internal/foreign/abi/fallback/FallbackLinker.java index 07f75e6919a..a8d3eea8718 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/fallback/FallbackLinker.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/fallback/FallbackLinker.java @@ -24,13 +24,6 @@ */ package jdk.internal.foreign.abi.fallback; -import jdk.internal.foreign.AbstractMemorySegmentImpl; -import jdk.internal.foreign.MemorySessionImpl; -import jdk.internal.foreign.abi.AbstractLinker; -import jdk.internal.foreign.abi.CapturableState; -import jdk.internal.foreign.abi.LinkerOptions; -import jdk.internal.foreign.abi.SharedUtils; - import java.lang.foreign.AddressLayout; import java.lang.foreign.Arena; import java.lang.foreign.FunctionDescriptor; @@ -39,16 +32,6 @@ import java.lang.foreign.MemoryLayout; import java.lang.foreign.MemorySegment; import java.lang.foreign.SegmentAllocator; import java.lang.foreign.ValueLayout; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.ref.Reference; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - import static java.lang.foreign.ValueLayout.ADDRESS; import static java.lang.foreign.ValueLayout.JAVA_BOOLEAN; import static java.lang.foreign.ValueLayout.JAVA_BYTE; @@ -58,7 +41,21 @@ import static java.lang.foreign.ValueLayout.JAVA_FLOAT; import static java.lang.foreign.ValueLayout.JAVA_INT; import static java.lang.foreign.ValueLayout.JAVA_LONG; import static java.lang.foreign.ValueLayout.JAVA_SHORT; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; import static java.lang.invoke.MethodHandles.foldArguments; +import java.lang.invoke.MethodType; +import java.lang.ref.Reference; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import jdk.internal.foreign.AbstractMemorySegmentImpl; +import jdk.internal.foreign.MemorySessionImpl; +import jdk.internal.foreign.abi.AbstractLinker; +import jdk.internal.foreign.abi.CapturableState; +import jdk.internal.foreign.abi.LinkerOptions; +import jdk.internal.foreign.abi.SharedUtils; public final class FallbackLinker extends AbstractLinker { @@ -95,7 +92,7 @@ public final class FallbackLinker extends AbstractLinker { .mapToInt(CapturableState::mask) .reduce(0, (a, b) -> a | b); DowncallData invData = new DowncallData(cif, function.returnLayout().orElse(null), - function.argumentLayouts(), capturedStateMask); + function.argumentLayouts(), capturedStateMask, options.allowsHeapAccess()); MethodHandle target = MethodHandles.insertArguments(MH_DO_DOWNCALL, 2, invData); @@ -155,12 +152,13 @@ public final class FallbackLinker extends AbstractLinker { } private record DowncallData(MemorySegment cif, MemoryLayout returnLayout, List argLayouts, - int capturedStateMask) {} + int capturedStateMask, boolean allowsHeapAccess) {} private static Object doDowncall(SegmentAllocator returnAllocator, Object[] args, DowncallData invData) { List acquiredSessions = new ArrayList<>(); try (Arena arena = Arena.ofConfined()) { int argStart = 0; + Object[] heapBases = invData.allowsHeapAccess() ? new Object[args.length] : null; MemorySegment target = (MemorySegment) args[argStart++]; MemorySessionImpl targetImpl = ((AbstractMemorySegmentImpl) target).sessionImpl(); @@ -180,12 +178,22 @@ public final class FallbackLinker extends AbstractLinker { for (int i = 0; i < argLayouts.size(); i++) { Object arg = args[argStart + i]; MemoryLayout layout = argLayouts.get(i); - MemorySegment argSeg = arena.allocate(layout); - writeValue(arg, layout, argSeg, addr -> { - MemorySessionImpl sessionImpl = ((AbstractMemorySegmentImpl) addr).sessionImpl(); + + if (layout instanceof AddressLayout) { + AbstractMemorySegmentImpl ms = (AbstractMemorySegmentImpl) arg; + MemorySessionImpl sessionImpl = ms.sessionImpl(); sessionImpl.acquire0(); acquiredSessions.add(sessionImpl); - }); + if (invData.allowsHeapAccess() && !ms.isNative()) { + heapBases[i] = ms.unsafeGetBase(); + // write the offset to the arg segment, add array ptr to it in native code + layout = JAVA_LONG; + arg = ms.address(); + } + } + + MemorySegment argSeg = arena.allocate(layout); + writeValue(arg, layout, argSeg); argPtrs.setAtIndex(ADDRESS, i, argSeg); } @@ -194,7 +202,8 @@ public final class FallbackLinker extends AbstractLinker { retSeg = (invData.returnLayout() instanceof GroupLayout ? returnAllocator : arena).allocate(invData.returnLayout); } - LibFallback.doDowncall(invData.cif, target, retSeg, argPtrs, capturedState, invData.capturedStateMask()); + LibFallback.doDowncall(invData.cif, target, retSeg, argPtrs, capturedState, invData.capturedStateMask(), + heapBases, args.length); Reference.reachabilityFence(invData.cif()); @@ -235,11 +244,6 @@ public final class FallbackLinker extends AbstractLinker { // where private static void writeValue(Object arg, MemoryLayout layout, MemorySegment argSeg) { - writeValue(arg, layout, argSeg, addr -> {}); - } - - private static void writeValue(Object arg, MemoryLayout layout, MemorySegment argSeg, - Consumer acquireCallback) { switch (layout) { case ValueLayout.OfBoolean bl -> argSeg.set(bl, 0, (Boolean) arg); case ValueLayout.OfByte bl -> argSeg.set(bl, 0, (Byte) arg); @@ -249,11 +253,7 @@ public final class FallbackLinker extends AbstractLinker { case ValueLayout.OfLong ll -> argSeg.set(ll, 0, (Long) arg); case ValueLayout.OfFloat fl -> argSeg.set(fl, 0, (Float) arg); case ValueLayout.OfDouble dl -> argSeg.set(dl, 0, (Double) arg); - case AddressLayout al -> { - MemorySegment addrArg = (MemorySegment) arg; - acquireCallback.accept(addrArg); - argSeg.set(al, 0, addrArg); - } + case AddressLayout al -> argSeg.set(al, 0, (MemorySegment) arg); case GroupLayout _ -> MemorySegment.copy((MemorySegment) arg, 0, argSeg, 0, argSeg.byteSize()); // by-value struct case null, default -> { diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/fallback/LibFallback.java b/src/java.base/share/classes/jdk/internal/foreign/abi/fallback/LibFallback.java index 8e3e29d6a4a..106850a3042 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/fallback/LibFallback.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/fallback/LibFallback.java @@ -93,10 +93,12 @@ final class LibFallback { * @see jdk.internal.foreign.abi.CapturableState */ static void doDowncall(MemorySegment cif, MemorySegment target, MemorySegment retPtr, MemorySegment argPtrs, - MemorySegment capturedState, int capturedStateMask) { + MemorySegment capturedState, int capturedStateMask, + Object[] heapBases, int numArgs) { doDowncall(cif.address(), target.address(), - retPtr == null ? 0 : retPtr.address(), argPtrs.address(), - capturedState == null ? 0 : capturedState.address(), capturedStateMask); + retPtr == null ? 0 : retPtr.address(), argPtrs.address(), + capturedState == null ? 0 : capturedState.address(), capturedStateMask, + heapBases, numArgs); } /** @@ -210,7 +212,9 @@ final class LibFallback { private static native int createClosure(long cif, Object userData, long[] ptrs); private static native void freeClosure(long closureAddress, long globalTarget); - private static native void doDowncall(long cif, long fn, long rvalue, long avalues, long capturedState, int capturedStateMask); + private static native void doDowncall(long cif, long fn, long rvalue, long avalues, + long capturedState, int capturedStateMask, + Object[] heapBases, int numArgs); private static native int ffi_prep_cif(long cif, int abi, int nargs, long rtype, long atypes); private static native int ffi_prep_cif_var(long cif, int abi, int nfixedargs, int ntotalargs, long rtype, long atypes); diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/ppc64/CallArranger.java b/src/java.base/share/classes/jdk/internal/foreign/abi/ppc64/CallArranger.java index dd92b4130a4..d07cd79e548 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/ppc64/CallArranger.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/ppc64/CallArranger.java @@ -110,8 +110,8 @@ public abstract class CallArranger { public Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall, LinkerOptions options) { CallingSequenceBuilder csb = new CallingSequenceBuilder(C, forUpcall, options); - BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true); - BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false); + BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess()); + BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false); boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout()); if (returnInMemory) { @@ -343,8 +343,11 @@ public abstract class CallArranger { // Compute recipe for transfering arguments / return values to C from Java. class UnboxBindingCalculator extends BindingCalculator { - UnboxBindingCalculator(boolean forArguments) { + private final boolean useAddressPairs; + + UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) { super(forArguments); + this.useAddressPairs = useAddressPairs; } @Override @@ -411,8 +414,16 @@ public abstract class CallArranger { } case POINTER -> { VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER, false); - bindings.unboxAddress() - .vmStore(storage, long.class); + if (useAddressPairs) { + bindings.dup() + .segmentBase() + .vmStore(storage, Object.class) + .segmentOffsetAllowHeap() + .vmStore(null, long.class); + } else { + bindings.unboxAddress(); + bindings.vmStore(storage, long.class); + } } case INTEGER -> { // ABI requires all int types to get extended to 64 bit. diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/riscv64/linux/LinuxRISCV64CallArranger.java b/src/java.base/share/classes/jdk/internal/foreign/abi/riscv64/linux/LinuxRISCV64CallArranger.java index 60959cf7544..e8ac33054d2 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/riscv64/linux/LinuxRISCV64CallArranger.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/riscv64/linux/LinuxRISCV64CallArranger.java @@ -85,8 +85,8 @@ public class LinuxRISCV64CallArranger { public static Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall, LinkerOptions options) { CallingSequenceBuilder csb = new CallingSequenceBuilder(CLinux, forUpcall, options); - BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true); - BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false); + BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess()); + BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false); boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout()); if (returnInMemory) { @@ -252,11 +252,13 @@ public class LinuxRISCV64CallArranger { } static final class UnboxBindingCalculator extends BindingCalculator { - final boolean forArguments; + protected final boolean forArguments; + private final boolean useAddressPairs; - UnboxBindingCalculator(boolean forArguments) { + UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) { super(forArguments); this.forArguments = forArguments; + this.useAddressPairs = useAddressPairs; } @Override @@ -280,9 +282,17 @@ public class LinuxRISCV64CallArranger { bindings.vmStore(storage, carrier); } case POINTER -> { - bindings.unboxAddress(); VMStorage storage = storageCalculator.getStorage(StorageType.INTEGER); - bindings.vmStore(storage, long.class); + if (useAddressPairs) { + bindings.dup() + .segmentBase() + .vmStore(storage, Object.class) + .segmentOffsetAllowHeap() + .vmStore(null, long.class); + } else { + bindings.unboxAddress(); + bindings.vmStore(storage, long.class); + } } case STRUCT_REGISTER_X -> { assert carrier == MemorySegment.class; diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/s390/linux/LinuxS390CallArranger.java b/src/java.base/share/classes/jdk/internal/foreign/abi/s390/linux/LinuxS390CallArranger.java index 009cfb6227f..7e6c254079a 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/s390/linux/LinuxS390CallArranger.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/s390/linux/LinuxS390CallArranger.java @@ -82,8 +82,8 @@ public class LinuxS390CallArranger { public static Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall, LinkerOptions options) { CallingSequenceBuilder csb = new CallingSequenceBuilder(CLinux, forUpcall, options); - BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true); - BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false); + BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess()); + BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false); boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout()); if (returnInMemory) { @@ -199,8 +199,11 @@ public class LinuxS390CallArranger { // Compute recipe for transferring arguments / return values to C from Java. static class UnboxBindingCalculator extends BindingCalculator { - UnboxBindingCalculator(boolean forArguments) { + private final boolean useAddressPairs; + + UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) { super(forArguments); + this.useAddressPairs = useAddressPairs; } @Override @@ -231,8 +234,16 @@ public class LinuxS390CallArranger { } case POINTER -> { VMStorage storage = storageCalculator.getStorage(StorageType.INTEGER, false); - bindings.unboxAddress() - .vmStore(storage, long.class); + if (useAddressPairs) { + bindings.dup() + .segmentBase() + .vmStore(storage, Object.class) + .segmentOffsetAllowHeap() + .vmStore(null, long.class); + } else { + bindings.unboxAddress(); + bindings.vmStore(storage, long.class); + } } case INTEGER -> { // ABI requires all int types to get extended to 64 bit. diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/x64/sysv/CallArranger.java b/src/java.base/share/classes/jdk/internal/foreign/abi/x64/sysv/CallArranger.java index 9dadc54b53b..78db1ca89e6 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/x64/sysv/CallArranger.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/x64/sysv/CallArranger.java @@ -97,8 +97,8 @@ public class CallArranger { public static Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall, LinkerOptions options) { CallingSequenceBuilder csb = new CallingSequenceBuilder(CSysV, forUpcall, options); - BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true); - BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false); + BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess()); + BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false); boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout()); if (returnInMemory) { @@ -246,9 +246,11 @@ public class CallArranger { } static class UnboxBindingCalculator extends BindingCalculator { + private final boolean useAddressPairs; - UnboxBindingCalculator(boolean forArguments) { + UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) { super(forArguments); + this.useAddressPairs = useAddressPairs; } @Override @@ -275,10 +277,18 @@ public class CallArranger { } } case POINTER -> { - bindings.unboxAddress(); VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER); - bindings.vmStore(storage, long.class); - } + if (useAddressPairs) { + bindings.dup() + .segmentBase() + .vmStore(storage, Object.class) + .segmentOffsetAllowHeap() + .vmStore(null, long.class); + } else { + bindings.unboxAddress(); + bindings.vmStore(storage, long.class); + } + } case INTEGER -> { VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER); bindings.vmStore(storage, carrier); diff --git a/src/java.base/share/classes/jdk/internal/foreign/abi/x64/windows/CallArranger.java b/src/java.base/share/classes/jdk/internal/foreign/abi/x64/windows/CallArranger.java index dc66ba9832c..cc69e79a092 100644 --- a/src/java.base/share/classes/jdk/internal/foreign/abi/x64/windows/CallArranger.java +++ b/src/java.base/share/classes/jdk/internal/foreign/abi/x64/windows/CallArranger.java @@ -87,9 +87,9 @@ public class CallArranger { class CallingSequenceBuilderHelper { final CallingSequenceBuilder csb = new CallingSequenceBuilder(CWindows, forUpcall, options); final BindingCalculator argCalc = - forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true); + forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true, options.allowsHeapAccess()); final BindingCalculator retCalc = - forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false); + forUpcall ? new UnboxBindingCalculator(false, false) : new BoxBindingCalculator(false); void addArgumentBindings(Class carrier, MemoryLayout layout, boolean isVararg) { csb.addArgumentBindings(carrier, layout, argCalc.getBindings(carrier, layout, isVararg)); @@ -184,9 +184,12 @@ public class CallArranger { static class UnboxBindingCalculator implements BindingCalculator { private final StorageCalculator storageCalculator; + private final boolean useAddressPairs; - UnboxBindingCalculator(boolean forArguments) { + UnboxBindingCalculator(boolean forArguments, boolean useAddressPairs) { this.storageCalculator = new StorageCalculator(forArguments); + assert !useAddressPairs || forArguments; + this.useAddressPairs = useAddressPairs; } @Override @@ -209,9 +212,17 @@ public class CallArranger { bindings.vmStore(storage, long.class); } case POINTER -> { - bindings.unboxAddress(); VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER); - bindings.vmStore(storage, long.class); + if (useAddressPairs) { + bindings.dup() + .segmentBase() + .vmStore(storage, Object.class) + .segmentOffsetAllowHeap() + .vmStore(null, long.class); + } else { + bindings.unboxAddress(); + bindings.vmStore(storage, long.class); + } } case INTEGER -> { VMStorage storage = storageCalculator.nextStorage(StorageType.INTEGER); diff --git a/src/java.base/share/native/libfallbackLinker/fallbackLinker.c b/src/java.base/share/native/libfallbackLinker/fallbackLinker.c index 41f4edbe54e..5c7531dcd75 100644 --- a/src/java.base/share/native/libfallbackLinker/fallbackLinker.c +++ b/src/java.base/share/native/libfallbackLinker/fallbackLinker.c @@ -28,6 +28,7 @@ #include #include +#include #include #include #ifdef _WIN64 @@ -109,9 +110,39 @@ static void do_capture_state(int32_t* value_ptr, int captured_state_mask) { } JNIEXPORT void JNICALL -Java_jdk_internal_foreign_abi_fallback_LibFallback_doDowncall(JNIEnv* env, jclass cls, jlong cif, jlong fn, jlong rvalue, jlong avalues, jlong jcaptured_state, jint captured_state_mask) { +Java_jdk_internal_foreign_abi_fallback_LibFallback_doDowncall(JNIEnv* env, jclass cls, jlong cif, jlong fn, jlong rvalue, + jlong avalues, jlong jcaptured_state, jint captured_state_mask, + jarray heapBases, jint numArgs) { + void** carrays; + if (heapBases != NULL) { + void** aptrs = jlong_to_ptr(avalues); + carrays = malloc(sizeof(void*) * numArgs); + for (int i = 0; i < numArgs; i++) { + jarray hb = (jarray) (*env)->GetObjectArrayElement(env, heapBases, i); + if (hb != NULL) { + // *(aptrs[i]) is the offset into the segment (from MS::address) + // we add the base address of the array to it here + jboolean isCopy; + jbyte* arrayPtr = (*env)->GetPrimitiveArrayCritical(env, hb, &isCopy); + carrays[i] = arrayPtr; + int offset = *((int*)aptrs[i]); + *((void**)aptrs[i]) = arrayPtr + offset; + } + } + } + ffi_call(jlong_to_ptr(cif), jlong_to_ptr(fn), jlong_to_ptr(rvalue), jlong_to_ptr(avalues)); + if (heapBases != NULL) { + for (int i = 0; i < numArgs; i++) { + jarray hb = (jarray) (*env)->GetObjectArrayElement(env, heapBases, i); + if (hb != NULL) { + (*env)->ReleasePrimitiveArrayCritical(env, hb, carrays[i], JNI_COMMIT); + } + } + free(carrays); + } + if (captured_state_mask != 0) { int32_t* captured_state = jlong_to_ptr(jcaptured_state); do_capture_state(captured_state, captured_state_mask); diff --git a/test/jdk/java/foreign/NativeTestHelper.java b/test/jdk/java/foreign/NativeTestHelper.java index 24c9db3b59f..d12593f7b85 100644 --- a/test/jdk/java/foreign/NativeTestHelper.java +++ b/test/jdk/java/foreign/NativeTestHelper.java @@ -195,7 +195,9 @@ public class NativeTestHelper { return result; } - public record TestValue (Object value, Consumer check) {} + public record TestValue (Object value, Consumer check) { + public void check(Object actual) { check.accept(actual); } + } public static TestValue genTestValue(MemoryLayout layout, SegmentAllocator allocator) { return genTestValue(DEFAULT_RANDOM, layout, allocator); @@ -237,6 +239,9 @@ public class NativeTestHelper { } else if (layout instanceof ValueLayout.OfShort) { short value = (short) random.nextInt(); return new TestValue(value, actual -> assertEquals(actual, value)); + } else if (layout instanceof ValueLayout.OfChar) { + char value = (char) random.nextInt(); + return new TestValue(value, actual -> assertEquals(actual, value)); } else if (layout instanceof ValueLayout.OfInt) { int value = random.nextInt(); return new TestValue(value, actual -> assertEquals(actual, value)); diff --git a/test/jdk/java/foreign/TestIllegalLink.java b/test/jdk/java/foreign/TestIllegalLink.java index 33bd8c5f6b5..b5052b4346c 100644 --- a/test/jdk/java/foreign/TestIllegalLink.java +++ b/test/jdk/java/foreign/TestIllegalLink.java @@ -102,7 +102,7 @@ public class TestIllegalLink extends NativeTestHelper { return new Object[][]{ { Linker.Option.firstVariadicArg(0) }, { Linker.Option.captureCallState("errno") }, - { Linker.Option.critical() }, + { Linker.Option.critical(false) }, }; } @@ -194,7 +194,7 @@ public class TestIllegalLink extends NativeTestHelper { }, { FunctionDescriptor.ofVoid(), - new Linker.Option[]{Linker.Option.critical(), Linker.Option.captureCallState("errno")}, + new Linker.Option[]{Linker.Option.critical(false), Linker.Option.captureCallState("errno")}, "Incompatible linker options: captureCallState, critical" }, })); diff --git a/test/jdk/java/foreign/critical/TestCritical.java b/test/jdk/java/foreign/critical/TestCritical.java new file mode 100644 index 00000000000..ab5d6017aba --- /dev/null +++ b/test/jdk/java/foreign/critical/TestCritical.java @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @library ../ /test/lib + * + * @run testng/othervm --enable-native-access=ALL-UNNAMED TestCritical + */ + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemoryLayout; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.lang.foreign.SequenceLayout; +import java.lang.foreign.StructLayout; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.VarHandle; +import java.util.ArrayList; +import java.util.List; +import java.util.function.IntFunction; +import java.util.stream.Stream; + +import static org.testng.Assert.assertEquals; + +public class TestCritical extends NativeTestHelper { + + static { + System.loadLibrary("Critical"); + } + + @Test + public void testEmpty() throws Throwable { + MethodHandle handle = downcallHandle("empty", FunctionDescriptor.ofVoid(), Linker.Option.critical(false)); + handle.invokeExact(); + } + + @Test + public void testIdentity() throws Throwable { + MethodHandle handle = downcallHandle("identity", FunctionDescriptor.of(C_INT, C_INT), Linker.Option.critical(false)); + int result = (int) handle.invokeExact(42); + assertEquals(result, 42); + } + + @Test + public void testWithReturnBuffer() throws Throwable { + StructLayout bigLayout = MemoryLayout.structLayout( + C_LONG_LONG.withName("x"), + C_LONG_LONG.withName("y")); + + MethodHandle handle = downcallHandle("with_return_buffer", FunctionDescriptor.of(bigLayout), Linker.Option.critical(false)); + VarHandle vhX = bigLayout.varHandle(MemoryLayout.PathElement.groupElement("x")); + VarHandle vhY = bigLayout.varHandle(MemoryLayout.PathElement.groupElement("y")); + try (Arena arena = Arena.ofConfined()) { + MemorySegment result = (MemorySegment) handle.invokeExact((SegmentAllocator) arena); + long x = (long) vhX.get(result, 0L); + assertEquals(x, 10); + long y = (long) vhY.get(result, 0L); + assertEquals(y, 11); + } + } + + public record AllowHeapCase(IntFunction newArraySegment, ValueLayout elementLayout, + String fName, FunctionDescriptor fDesc, boolean readOnly) {} + + @Test(dataProvider = "allowHeapCases") + public void testAllowHeap(AllowHeapCase testCase) throws Throwable { + MethodHandle handle = downcallHandle(testCase.fName(), testCase.fDesc(), Linker.Option.critical(true)); + int elementCount = 10; + MemorySegment heapSegment = testCase.newArraySegment().apply(elementCount); + if (testCase.readOnly()) { + heapSegment = heapSegment.asReadOnly(); + } + SequenceLayout sequence = MemoryLayout.sequenceLayout(elementCount, testCase.elementLayout()); + + try (Arena arena = Arena.ofConfined()) { + TestValue[] tvs = genTestArgs(testCase.fDesc(), arena); + Object[] args = Stream.of(tvs).map(TestValue::value).toArray(); + + // inject our custom last three arguments + args[args.length - 1] = (int) sequence.byteSize(); + TestValue sourceSegment = genTestValue(sequence, arena); + args[args.length - 2] = sourceSegment.value(); + args[args.length - 3] = heapSegment; + + if (handle.type().parameterType(0) == SegmentAllocator.class) { + Object[] newArgs = new Object[args.length + 1]; + newArgs[0] = arena; + System.arraycopy(args, 0, newArgs, 1, args.length); + args = newArgs; + } + + Object o = handle.invokeWithArguments(args); + + if (o != null) { + tvs[0].check(o); + } + + // check that writes went through to array + sourceSegment.check(heapSegment); + } + } + + @DataProvider + public Object[][] allowHeapCases() { + FunctionDescriptor voidDesc = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT); + FunctionDescriptor intDesc = voidDesc.changeReturnLayout(C_INT).insertArgumentLayouts(0, C_INT); + StructLayout L2 = MemoryLayout.structLayout( + C_LONG_LONG.withName("x"), + C_LONG_LONG.withName("y") + ); + FunctionDescriptor L2Desc = voidDesc.changeReturnLayout(L2).insertArgumentLayouts(0, L2); + StructLayout L3 = MemoryLayout.structLayout( + C_LONG_LONG.withName("x"), + C_LONG_LONG.withName("y"), + C_LONG_LONG.withName("z") + ); + FunctionDescriptor L3Desc = voidDesc.changeReturnLayout(L3).insertArgumentLayouts(0, L3); + FunctionDescriptor stackDesc = voidDesc.insertArgumentLayouts(0, + C_LONG_LONG, C_LONG_LONG, C_LONG_LONG, C_LONG_LONG, + C_LONG_LONG, C_LONG_LONG, C_LONG_LONG, C_LONG_LONG, + C_CHAR, C_SHORT, C_INT); + + List cases = new ArrayList<>(); + + for (HeapSegmentFactory hsf : HeapSegmentFactory.values()) { + cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_void", voidDesc, false)); + cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_int", intDesc, false)); + cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_return_buffer", L2Desc, false)); + cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_imr", L3Desc, false)); + cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_void_stack", stackDesc, false)); + // readOnly + cases.add(new AllowHeapCase(hsf.newArray, hsf.elementLayout, "test_allow_heap_void", voidDesc, true)); + } + + return cases.stream().map(e -> new Object[]{ e }).toArray(Object[][]::new); + } + + private enum HeapSegmentFactory { + BYTE(i -> MemorySegment.ofArray(new byte[i]), ValueLayout.JAVA_BYTE), + SHORT(i -> MemorySegment.ofArray(new short[i]), ValueLayout.JAVA_SHORT), + CHAR(i -> MemorySegment.ofArray(new char[i]), ValueLayout.JAVA_CHAR), + INT(i -> MemorySegment.ofArray(new int[i]), ValueLayout.JAVA_INT), + LONG(i -> MemorySegment.ofArray(new long[i]), ValueLayout.JAVA_LONG), + FLOAT(i -> MemorySegment.ofArray(new float[i]), ValueLayout.JAVA_FLOAT), + DOUBLE(i -> MemorySegment.ofArray(new double[i]), ValueLayout.JAVA_DOUBLE); + + IntFunction newArray; + ValueLayout elementLayout; + + private HeapSegmentFactory(IntFunction newArray, ValueLayout elementLayout) { + this.newArray = newArray; + this.elementLayout = elementLayout; + } + } +} diff --git a/test/jdk/java/foreign/trivial/TestCriticalUpcall.java b/test/jdk/java/foreign/critical/TestCriticalUpcall.java similarity index 97% rename from test/jdk/java/foreign/trivial/TestCriticalUpcall.java rename to test/jdk/java/foreign/critical/TestCriticalUpcall.java index 2172f0ceda5..fe19720bf93 100644 --- a/test/jdk/java/foreign/trivial/TestCriticalUpcall.java +++ b/test/jdk/java/foreign/critical/TestCriticalUpcall.java @@ -50,7 +50,7 @@ public class TestCriticalUpcall extends UpcallTestHelper { public static void main(String[] args) throws Throwable { System.loadLibrary("Critical"); - MethodHandle mh = downcallHandle("do_upcall", FunctionDescriptor.ofVoid(C_POINTER), Linker.Option.critical()); + MethodHandle mh = downcallHandle("do_upcall", FunctionDescriptor.ofVoid(C_POINTER), Linker.Option.critical(false)); MemorySegment stub = upcallStub(Runner.class, "target", FunctionDescriptor.ofVoid()); mh.invokeExact(stub); } diff --git a/test/jdk/java/foreign/critical/TestStressAllowHeap.java b/test/jdk/java/foreign/critical/TestStressAllowHeap.java new file mode 100644 index 00000000000..4dd3b70c01f --- /dev/null +++ b/test/jdk/java/foreign/critical/TestStressAllowHeap.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @library ../ /test/lib + * @requires vm.debug + * @run main/othervm + * -Xms1g -Xmx1g + * -XX:+CheckUnhandledOops + * -Xlog:gc -Xlog:gc+jni=debug + * --enable-native-access=ALL-UNNAMED + * TestStressAllowHeap + */ + +import java.lang.foreign.*; +import java.lang.invoke.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static jdk.test.lib.Asserts.*; + +/** + * Test verifies the GCLocker::lock_critical slow path with FFM. + * This is the case where we enter a critical section with _needs_gc == true, + * and need to take a safepoint. + * + * Based on gc/TestJNICriticalStressTest + */ +public class TestStressAllowHeap { + private static final long DURATION_SECONDS = 30; + + public static void main(String... args) throws Exception { + System.out.println("Running for " + DURATION_SECONDS + " secs"); + + int numCriticalThreads = Runtime.getRuntime().availableProcessors(); + System.out.println("Starting " + numCriticalThreads + " critical threads"); + for (int i = 0; i < numCriticalThreads; i += 1) { + Thread.ofPlatform().start(new CriticalWorker()); + } + + long durationMS = 1000L * DURATION_SECONDS; + try { + Thread.sleep(durationMS); + } catch (InterruptedException e) { + e.printStackTrace(); + System.exit(-1); + } + + // hitting the problematic code path doesn't seem to be guaranteed + // and we can not guarantee it by, e.g. stalling in a critical method + // since that can lock up the VM (and our test code) + } + + private static class CriticalWorker extends NativeTestHelper implements Runnable { + static { + System.loadLibrary("Critical"); + } + + private void doStep(MethodHandle handle, SequenceLayout sequence) throws Throwable { + try (Arena arena = Arena.ofConfined()) { + MemorySegment heapSegment = MemorySegment.ofArray(new int[(int) sequence.elementCount()]); + TestValue sourceSegment = genTestValue(sequence, arena); + + handle.invokeExact(heapSegment, (MemorySegment) sourceSegment.value(), (int) sequence.byteSize()); + + // check that writes went through to array + sourceSegment.check(heapSegment); + } + } + + @Override + public void run() { + FunctionDescriptor fDesc = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT); + MethodHandle handle = Linker.nativeLinker().downcallHandle( + SymbolLookup.loaderLookup().find("test_allow_heap_void").get(), + fDesc, + Linker.Option.critical(true)); + int elementCount = 10; + SequenceLayout sequence = MemoryLayout.sequenceLayout(elementCount, C_INT); + while (true) { + try { + doStep(handle, sequence); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + } + } +} diff --git a/test/jdk/java/foreign/critical/libCritical.c b/test/jdk/java/foreign/critical/libCritical.c new file mode 100644 index 00000000000..52bfa60677a --- /dev/null +++ b/test/jdk/java/foreign/critical/libCritical.c @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include + +#ifdef _WIN64 +#define EXPORT __declspec(dllexport) +#else +#define EXPORT +#endif + +EXPORT void empty() {} + +EXPORT int identity(int value) { + return value; +} + +// 128 bit struct returned in buffer on SysV +struct Big { + long long x; + long long y; +}; + +EXPORT struct Big with_return_buffer() { + struct Big b; + b.x = 10; + b.y = 11; + return b; +} + +EXPORT void do_upcall(void(*f)(void)) { + f(); +} + +// copy bytes into heap array +EXPORT void test_allow_heap_void(unsigned char* heapArr, unsigned char* nativeArr, int numBytes) { + for (int i = 0; i < numBytes; i++) { + heapArr[i] = nativeArr[i]; + } +} + +EXPORT int test_allow_heap_int(int a0, unsigned char* heapArr, unsigned char* nativeArr, int numBytes) { + for (int i = 0; i < numBytes; i++) { + heapArr[i] = nativeArr[i]; + } + return a0; +} + +struct L2 { + long long x; + long long y; +}; + +EXPORT struct L2 test_allow_heap_return_buffer(struct L2 a0, unsigned char* heapArr, unsigned char* nativeArr, int numBytes) { + for (int i = 0; i < numBytes; i++) { + heapArr[i] = nativeArr[i]; + } + return a0; +} + +struct L3 { + long long x; + long long y; + long long z; +}; + +EXPORT struct L3 test_allow_heap_imr(struct L3 a0, unsigned char* heapArr, unsigned char* nativeArr, int numBytes) { + for (int i = 0; i < numBytes; i++) { + heapArr[i] = nativeArr[i]; + } + return a0; +} + +// copy bytes into heap array +EXPORT void test_allow_heap_void_stack(long long a0, long long a1, long long a2, long long a3, long long a4, long long a5, + long long a6, long long a7, char c0, short s0, int i0, + unsigned char* heapArr, unsigned char* nativeArr, int numBytes) { + for (int i = 0; i < numBytes; i++) { + heapArr[i] = nativeArr[i]; + } +} diff --git a/test/jdk/java/foreign/largestub/TestLargeStub.java b/test/jdk/java/foreign/largestub/TestLargeStub.java index eca50c8f791..eaf82648778 100644 --- a/test/jdk/java/foreign/largestub/TestLargeStub.java +++ b/test/jdk/java/foreign/largestub/TestLargeStub.java @@ -36,20 +36,37 @@ import java.lang.foreign.MemoryLayout; import java.util.stream.Stream; public class TestLargeStub extends NativeTestHelper { + + MemoryLayout STRUCT_LL = MemoryLayout.structLayout( + C_LONG_LONG, + C_LONG_LONG + ); // 16 byte struct triggers return buffer usage on SysV + @Test public void testDowncall() { // Link a handle with a large number of arguments, to try and overflow the code buffer Linker.nativeLinker().downcallHandle( - FunctionDescriptor.of(C_LONG_LONG, - Stream.generate(() -> C_DOUBLE).limit(50).toArray(MemoryLayout[]::new)), + FunctionDescriptor.of(STRUCT_LL, + Stream.generate(() -> C_DOUBLE).limit(124).toArray(MemoryLayout[]::new)), Linker.Option.captureCallState("errno")); } + @Test + public void testDowncallAllowHeap() { + // Link a handle with a large number of address arguments, to try and overflow the code buffer + // Using 83 parameters should get us 255 parameter slots in total: + // 83 oops + 166 for offsets + 2 for the target address + 2 for return buffer + MH recv. + NEP + Linker.nativeLinker().downcallHandle( + FunctionDescriptor.of(STRUCT_LL, + Stream.generate(() -> C_POINTER).limit(83).toArray(MemoryLayout[]::new)), + Linker.Option.critical(true)); + } + @Test public void testUpcall() { // Link a handle with a large number of arguments, to try and overflow the code buffer Linker.nativeLinker().downcallHandle( - FunctionDescriptor.of(C_LONG_LONG, - Stream.generate(() -> C_DOUBLE).limit(50).toArray(MemoryLayout[]::new))); + FunctionDescriptor.of(STRUCT_LL, + Stream.generate(() -> C_DOUBLE).limit(125).toArray(MemoryLayout[]::new))); } } diff --git a/test/jdk/java/foreign/trivial/TestCritical.java b/test/jdk/java/foreign/trivial/TestCritical.java deleted file mode 100644 index df5783350a4..00000000000 --- a/test/jdk/java/foreign/trivial/TestCritical.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -/* - * @test - * @library ../ /test/lib - * @run testng/othervm --enable-native-access=ALL-UNNAMED TestCritical - */ - -import org.testng.annotations.Test; - -import java.lang.foreign.Arena; -import java.lang.foreign.FunctionDescriptor; -import java.lang.foreign.Linker; -import java.lang.foreign.MemoryLayout; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.SegmentAllocator; -import java.lang.foreign.StructLayout; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.VarHandle; - -import static org.testng.Assert.assertEquals; - -public class TestCritical extends NativeTestHelper { - - static { - System.loadLibrary("Critical"); - } - - @Test - public void testEmpty() throws Throwable { - MethodHandle handle = downcallHandle("empty", FunctionDescriptor.ofVoid(), Linker.Option.critical()); - handle.invokeExact(); - } - - @Test - public void testIdentity() throws Throwable { - MethodHandle handle = downcallHandle("identity", FunctionDescriptor.of(C_INT, C_INT), Linker.Option.critical()); - int result = (int) handle.invokeExact(42); - assertEquals(result, 42); - } - - @Test - public void testWithReturnBuffer() throws Throwable { - StructLayout bigLayout = MemoryLayout.structLayout( - C_LONG_LONG.withName("x"), - C_LONG_LONG.withName("y")); - - MethodHandle handle = downcallHandle("with_return_buffer", FunctionDescriptor.of(bigLayout), Linker.Option.critical()); - VarHandle vhX = bigLayout.varHandle(MemoryLayout.PathElement.groupElement("x")); - VarHandle vhY = bigLayout.varHandle(MemoryLayout.PathElement.groupElement("y")); - try (Arena arena = Arena.ofConfined()) { - MemorySegment result = (MemorySegment) handle.invokeExact((SegmentAllocator) arena); - long x = (long) vhX.get(result, 0L); - assertEquals(x, 10); - long y = (long) vhY.get(result, 0L); - assertEquals(y, 11); - } - } - -} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/CallOverheadHelper.java b/test/micro/org/openjdk/bench/java/lang/foreign/CallOverheadHelper.java index 2a305d26297..290d40f1202 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/CallOverheadHelper.java +++ b/test/micro/org/openjdk/bench/java/lang/foreign/CallOverheadHelper.java @@ -113,7 +113,7 @@ public class CallOverheadHelper extends CLayouts { MethodType mt = MethodType.methodType(void.class); FunctionDescriptor fd = FunctionDescriptor.ofVoid(); func_v = abi.downcallHandle(fd); - func_critical_v = abi.downcallHandle(fd, Linker.Option.critical()); + func_critical_v = abi.downcallHandle(fd, Linker.Option.critical(false)); func = insertArguments(func_v, 0, func_addr); func_critical = insertArguments(func_critical_v, 0, func_addr); } @@ -121,7 +121,7 @@ public class CallOverheadHelper extends CLayouts { identity_addr = loaderLibs.find("identity").orElseThrow(); FunctionDescriptor fd = FunctionDescriptor.of(C_INT, C_INT); identity_v = abi.downcallHandle(fd); - identity_critical_v = abi.downcallHandle(fd, Linker.Option.critical()); + identity_critical_v = abi.downcallHandle(fd, Linker.Option.critical(false)); identity = insertArguments(identity_v, 0, identity_addr); identity_critical = insertArguments(identity_critical_v, 0, identity_addr); } diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/CriticalCalls.java b/test/micro/org/openjdk/bench/java/lang/foreign/CriticalCalls.java new file mode 100644 index 00000000000..a7cfe6169a8 --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/CriticalCalls.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.bench.java.lang.foreign; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SegmentAllocator; +import java.lang.foreign.SymbolLookup; + +import java.lang.invoke.MethodHandle; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.concurrent.TimeUnit; + +import static java.lang.foreign.ValueLayout.*; + +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@State(org.openjdk.jmh.annotations.Scope.Thread) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(value = 3, jvmArgsAppend = { "--enable-native-access=ALL-UNNAMED" }) +public class CriticalCalls { + + static final MethodHandle PINNED; + static final MethodHandle NOT_PINNED; + + static { + System.loadLibrary("CriticalCalls"); + SymbolLookup lookup = SymbolLookup.loaderLookup(); + + MemorySegment sumIntsSym = lookup.find("sum_ints").get(); + FunctionDescriptor sumIntsDesc = FunctionDescriptor.of(JAVA_INT, ADDRESS, JAVA_INT); + + PINNED = Linker.nativeLinker().downcallHandle( + sumIntsSym, + sumIntsDesc, + Linker.Option.critical(true)); + NOT_PINNED = Linker.nativeLinker().downcallHandle( + sumIntsSym, + sumIntsDesc, + Linker.Option.critical(false)); + } + + @Param({"100", "10000", "1000000"}) + int size; + + int[] arr; + SegmentAllocator recycler; + + @Setup + public void setup() { + arr = new int[size]; + for (int i = 0; i < size; i++) { + arr[i] = i; + } + + recycler = SegmentAllocator.prefixAllocator(Arena.ofAuto().allocate(JAVA_INT, arr.length)); + } + + @Benchmark + public int callPinned() throws Throwable { + return (int) PINNED.invokeExact(MemorySegment.ofArray(arr), arr.length); + } + + @Benchmark + public int callNotPinned() throws Throwable { + try (Arena arena = Arena.ofConfined()) { + MemorySegment nativeArr = arena.allocateFrom(JAVA_INT, arr); + return (int) NOT_PINNED.invokeExact(nativeArr, arr.length); + } + } + + @Benchmark + public int callRecycled() throws Throwable { + MemorySegment nativeArr = recycler.allocateFrom(JAVA_INT, arr); + return (int) NOT_PINNED.invokeExact(nativeArr, arr.length); + } +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/Utils.java b/test/micro/org/openjdk/bench/java/lang/foreign/Utils.java index d8d04a35234..0bb7fa604a6 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/Utils.java +++ b/test/micro/org/openjdk/bench/java/lang/foreign/Utils.java @@ -28,7 +28,7 @@ import java.lang.reflect.Field; public class Utils { - static final Unsafe unsafe; + public static final Unsafe unsafe; //setup unsafe static { diff --git a/test/jdk/java/foreign/trivial/libCritical.c b/test/micro/org/openjdk/bench/java/lang/foreign/libCriticalCalls.c similarity index 76% rename from test/jdk/java/foreign/trivial/libCritical.c rename to test/micro/org/openjdk/bench/java/lang/foreign/libCriticalCalls.c index 8a825f9d42f..6c76ef3d1b3 100644 --- a/test/jdk/java/foreign/trivial/libCritical.c +++ b/test/micro/org/openjdk/bench/java/lang/foreign/libCriticalCalls.c @@ -21,33 +21,16 @@ * questions. */ -#include - #ifdef _WIN64 #define EXPORT __declspec(dllexport) #else #define EXPORT #endif -EXPORT void empty() {} - -EXPORT int identity(int value) { - return value; -} - -// 128 bit struct returned in buffer on SysV -struct Big { - long long x; - long long y; -}; - -EXPORT struct Big with_return_buffer() { - struct Big b; - b.x = 10; - b.y = 11; - return b; -} - -EXPORT void do_upcall(void(*f)(void)) { - f(); +EXPORT int sum_ints(int* arr, int size) { + int sum = 0; + for (int i = 0; i < size; i++) { + sum += arr[i]; + } + return sum; } diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/libToCString.c b/test/micro/org/openjdk/bench/java/lang/foreign/libToCString.c index 1fae0325dbc..8a8a1dd364c 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/libToCString.c +++ b/test/micro/org/openjdk/bench/java/lang/foreign/libToCString.c @@ -26,6 +26,8 @@ #include #include +#include "jlong.h" + JNIEXPORT jlong JNICALL Java_org_openjdk_bench_java_lang_foreign_ToCStringTest_writeString(JNIEnv *const env, const jclass cls, const jstring text) { const char *str = (*env)->GetStringUTFChars(env, text, NULL); jlong addr = ptr_to_jlong(str); diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/libToJavaString.c b/test/micro/org/openjdk/bench/java/lang/foreign/libToJavaString.c index ed89320574d..f4c407c1ab6 100644 --- a/test/micro/org/openjdk/bench/java/lang/foreign/libToJavaString.c +++ b/test/micro/org/openjdk/bench/java/lang/foreign/libToJavaString.c @@ -28,6 +28,8 @@ #include #include +#include "jlong.h" + JNIEXPORT jstring JNICALL Java_org_openjdk_bench_java_lang_foreign_ToJavaStringTest_readString(JNIEnv *const env, const jclass cls, jlong addr) { return (*env)->NewStringUTF(env, jlong_to_ptr(addr)); } diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayCriticalXorOpImpl.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayCriticalXorOpImpl.java new file mode 100644 index 00000000000..284bc64bd7a --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayCriticalXorOpImpl.java @@ -0,0 +1,11 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +public class GetArrayCriticalXorOpImpl implements XorOp { + + static { + System.loadLibrary("jnitest"); + } + + // Uses {Get|Release}PrimitiveArrayCritical to access the byte arrays + public native void xor(byte[] src, int sOff, byte[] dst, int dOff, int len); +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayElementsXorOpImpl.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayElementsXorOpImpl.java new file mode 100644 index 00000000000..df48c2e85af --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayElementsXorOpImpl.java @@ -0,0 +1,11 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +public class GetArrayElementsXorOpImpl implements XorOp { + + static { + System.loadLibrary("jnitest"); + } + + // Uses {Get|Release}ByteArrayElements to access the byte arrays + public native void xor(byte[] src, int sOff, byte[] dst, int dOff, int len); +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpCriticalImpl.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpCriticalImpl.java new file mode 100644 index 00000000000..250ba1157d7 --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpCriticalImpl.java @@ -0,0 +1,30 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.invoke.MethodHandle; + +import static java.lang.foreign.Linker.Option.critical; +import static org.openjdk.bench.java.lang.foreign.CLayouts.*; + +public class GetArrayForeignXorOpCriticalImpl implements XorOp { + + static { + System.loadLibrary("jnitest"); + + Linker linker; + linker = Linker.nativeLinker(); + FunctionDescriptor xor_op_func = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT); + xor_op = linker.downcallHandle(SymbolLookup.loaderLookup().find("xor_op").orElseThrow(), xor_op_func, critical(true)); + } + + static final MethodHandle xor_op; + GetArrayForeignXorOpCriticalImpl() { + } + + public void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable { + xor_op.invokeExact(MemorySegment.ofArray(src).asSlice(sOff), MemorySegment.ofArray(dst).asSlice(dOff), len); + } +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpImpl.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpImpl.java new file mode 100644 index 00000000000..e033127de6a --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpImpl.java @@ -0,0 +1,37 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.invoke.MethodHandle; + +import static java.lang.foreign.Linker.Option.critical; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.openjdk.bench.java.lang.foreign.CLayouts.*; + +public class GetArrayForeignXorOpImpl implements XorOp { + + static { + System.loadLibrary("jnitest"); + + Linker linker; + linker = Linker.nativeLinker(); + FunctionDescriptor xor_op_func = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT); + xor_op = linker.downcallHandle(SymbolLookup.loaderLookup().find("xor_op").orElseThrow(), xor_op_func, critical(false)); + } + + static final MethodHandle xor_op; + GetArrayForeignXorOpImpl() { + } + + public void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable { + try (Arena arena = Arena.ofConfined()) { + MemorySegment srcBuf = arena.allocateFrom(JAVA_BYTE, MemorySegment.ofArray(src), JAVA_BYTE, sOff, len); + MemorySegment dstBuf = arena.allocateFrom(JAVA_BYTE, MemorySegment.ofArray(dst), JAVA_BYTE, dOff, len); + xor_op.invokeExact(srcBuf, dstBuf, len); + MemorySegment.copy(dstBuf, JAVA_BYTE, 0, dst, dOff, len); + } + } +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpInitImpl.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpInitImpl.java new file mode 100644 index 00000000000..8a3df5d4f39 --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayForeignXorOpInitImpl.java @@ -0,0 +1,40 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.invoke.MethodHandle; + +import static java.lang.foreign.Linker.Option.critical; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.openjdk.bench.java.lang.foreign.CLayouts.C_INT; +import static org.openjdk.bench.java.lang.foreign.CLayouts.C_POINTER; + +public class GetArrayForeignXorOpInitImpl implements XorOp { + + static { + System.loadLibrary("jnitest"); + + Linker linker; + linker = Linker.nativeLinker(); + FunctionDescriptor xor_op_func = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT); + xor_op = linker.downcallHandle(SymbolLookup.loaderLookup().find("xor_op").orElseThrow(), xor_op_func, critical(false)); + } + + static final MethodHandle xor_op; + GetArrayForeignXorOpInitImpl() { + } + + public void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable { + try (Arena arena = Arena.ofConfined()) { + MemorySegment srcBuf = arena.allocate(len); + MemorySegment.copy(src, sOff, srcBuf, JAVA_BYTE, 0, len); + MemorySegment dstBuf = arena.allocate(len); + MemorySegment.copy(dst, dOff, dstBuf, JAVA_BYTE, 0, len); + xor_op.invokeExact(srcBuf, dstBuf, len); + MemorySegment.copy(dstBuf, JAVA_BYTE, 0, dst, dOff, len); + } + } +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayRegionXorOpImpl.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayRegionXorOpImpl.java new file mode 100644 index 00000000000..9104589405a --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayRegionXorOpImpl.java @@ -0,0 +1,11 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +public class GetArrayRegionXorOpImpl implements XorOp { + + static { + System.loadLibrary("jnitest"); + } + + // Uses {Get|Set}ByteArrayRegion to access the byte arrays + public native void xor(byte[] src, int sOff, byte[] dst, int dOff, int len); +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayUnsafeXorOpImpl.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayUnsafeXorOpImpl.java new file mode 100644 index 00000000000..4175dd03121 --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/GetArrayUnsafeXorOpImpl.java @@ -0,0 +1,47 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +import org.openjdk.bench.java.lang.foreign.Utils; +import sun.misc.Unsafe; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.invoke.MethodHandle; + +import static java.lang.foreign.Linker.Option.critical; +import static java.lang.foreign.ValueLayout.JAVA_BYTE; +import static org.openjdk.bench.java.lang.foreign.CLayouts.*; + +public class GetArrayUnsafeXorOpImpl implements XorOp { + + static final Unsafe UNSAFE = Utils.unsafe; + static final int BYTE_ARR_OFFSET = Utils.unsafe.arrayBaseOffset(byte[].class); + + static { + System.loadLibrary("jnitest"); + + Linker linker; + linker = Linker.nativeLinker(); + FunctionDescriptor xor_op_func = FunctionDescriptor.ofVoid(C_POINTER, C_POINTER, C_INT); + xor_op = linker.downcallHandle(SymbolLookup.loaderLookup().find("xor_op").orElseThrow(), xor_op_func, critical(false)); + } + + static final MethodHandle xor_op; + GetArrayUnsafeXorOpImpl() { + } + + public void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable { + long srcBuf = UNSAFE.allocateMemory(len); + long dstBuf = UNSAFE.allocateMemory(len); + UNSAFE.copyMemory(src, sOff + BYTE_ARR_OFFSET, null, srcBuf, len); + UNSAFE.copyMemory(dst, dOff + BYTE_ARR_OFFSET, null, dstBuf, len); + xorOp(srcBuf, dstBuf, len); + UNSAFE.copyMemory(null, dstBuf, dst, dOff + BYTE_ARR_OFFSET, len); + UNSAFE.freeMemory(srcBuf); + UNSAFE.freeMemory(dstBuf); + } + + native void xorOp(long src, long dst, int len); +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/XorOp.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/XorOp.java new file mode 100644 index 00000000000..4b3927e28c0 --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/XorOp.java @@ -0,0 +1,7 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +public interface XorOp { + + void xor(byte[] src, int sOff, byte[] dst, int dOff, int len) throws Throwable; + +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/XorTest.java b/test/micro/org/openjdk/bench/java/lang/foreign/xor/XorTest.java new file mode 100644 index 00000000000..42fa54e8c72 --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/XorTest.java @@ -0,0 +1,130 @@ +package org.openjdk.bench.java.lang.foreign.xor; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@State(org.openjdk.jmh.annotations.Scope.Thread) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Fork(value = 3, jvmArgsAppend = { "--enable-native-access=ALL-UNNAMED" }) + +public class XorTest { + + XorOp impl = null; + int alen; + int off; + int len; + byte[] src, dst; + + @Param + SizeKind sizeKind; + + @Param + ArrayKind arrayKind; + + public enum SizeKind { + SMALL, + MEDIUM, + LARGE; + } + + public enum ArrayKind { + JNI_ELEMENTS, + JNI_REGION, + JNI_CRITICAL, + FOREIGN_NO_INIT, + FOREIGN_INIT, + FOREIGN_CRITICAL, + UNSAFE; + } + + @Setup + public void setup() throws Throwable { + switch (arrayKind) { + case JNI_CRITICAL: + impl = new GetArrayCriticalXorOpImpl(); + break; + case JNI_ELEMENTS: + impl = new GetArrayElementsXorOpImpl(); + break; + case JNI_REGION: + impl = new GetArrayRegionXorOpImpl(); + break; + case FOREIGN_NO_INIT: + impl = new GetArrayForeignXorOpImpl(); + break; + case FOREIGN_INIT: + impl = new GetArrayForeignXorOpInitImpl(); + break; + case FOREIGN_CRITICAL: + impl = new GetArrayForeignXorOpCriticalImpl(); + break; + case UNSAFE: + impl = new GetArrayUnsafeXorOpImpl(); + break; + default: + throw new UnsupportedOperationException(arrayKind.toString()); + } + + switch (sizeKind) { + case SMALL: + alen = 1048576; // 1 MB + off = 1024 * 10; + len = 1024 * 100; // 100 KB + break; + case MEDIUM: + alen = 1048576 * 8; // 8 MB + off = 1048576 * 1; + len = 1048576 * 2; // 2 MB + break; + case LARGE: + alen = 1048576 * 100; // 100 MB + off = 1048576 * 5; + len = 1048576 * 10; // 10 MB + break; + default: + throw new UnsupportedOperationException(sizeKind.toString()); + } + + src = new byte[alen]; + dst = new byte[alen]; + Arrays.fill(src, off, off + len, (byte)0xaa); + Arrays.fill(dst, off, off + len, (byte)0x5a); + check(); + } + + void check() throws Throwable { + impl.xor(src, off, dst, off, len); + if (!verify(dst, off, off + len, (byte)0xf0)) { + throw new IllegalStateException("Failed to verify"); + } + } + + + @Benchmark + public void xor() throws Throwable { + impl.xor(src, off, dst, off, len); + } + + static boolean verify(byte[] buf, int start, int end, byte val) { + for (int i = start; i < end; ++i) { + if (buf[i] != val) + return false; + } + return true; + } + +} diff --git a/test/micro/org/openjdk/bench/java/lang/foreign/xor/libjnitest.c b/test/micro/org/openjdk/bench/java/lang/foreign/xor/libjnitest.c new file mode 100644 index 00000000000..2b74dc9c3d7 --- /dev/null +++ b/test/micro/org/openjdk/bench/java/lang/foreign/xor/libjnitest.c @@ -0,0 +1,75 @@ +#include +#include + +JNIEXPORT void xor_op(jbyte *restrict src, jbyte *restrict dst, jint len) { + for (int i = 0; i < len; ++i) { + dst[i] ^= src[i]; + } +} + +/* + * Class: com_oracle_jnitest_GetArrayCriticalXorOpImpl + * Method: xor + * Signature: ([BI[BII)V + */ +JNIEXPORT void JNICALL Java_org_openjdk_bench_java_lang_foreign_xor_GetArrayCriticalXorOpImpl_xor + (JNIEnv *env, jobject obj, jbyteArray src, jint sOff, jbyteArray dst, jint dOff, jint len) { + jbyte *sbuf = NULL; + jbyte *dbuf = NULL; + jboolean sIsCopy = JNI_FALSE; + jboolean dIsCopy = JNI_FALSE; + + sbuf = (*env)->GetPrimitiveArrayCritical(env, src, &sIsCopy); + dbuf = (*env)->GetPrimitiveArrayCritical(env, dst, &dIsCopy); + xor_op(&sbuf[sOff], &dbuf[dOff], len); + (*env)->ReleasePrimitiveArrayCritical(env, dst, dbuf, 0); + (*env)->ReleasePrimitiveArrayCritical(env, src, sbuf, JNI_ABORT); +} + +/* + * Class: com_oracle_jnitest_GetArrayElementsXorOpImpl + * Method: xor + * Signature: ([BI[BII)V + */ +JNIEXPORT void JNICALL Java_org_openjdk_bench_java_lang_foreign_xor_GetArrayElementsXorOpImpl_xor + (JNIEnv *env, jobject obj, jbyteArray src, jint sOff, jbyteArray dst, jint dOff, jint len) { + jbyte *sbuf = NULL; + jbyte *dbuf = NULL; + jboolean sIsCopy = JNI_FALSE; + jboolean dIsCopy = JNI_FALSE; + + sbuf = (*env)->GetByteArrayElements(env, src, &sIsCopy); + dbuf = (*env)->GetByteArrayElements(env, dst, &dIsCopy); + xor_op(&sbuf[sOff], &dbuf[dOff], len); + (*env)->ReleaseByteArrayElements(env, dst, dbuf, 0); + (*env)->ReleaseByteArrayElements(env, src, sbuf, JNI_ABORT); +} + +/* + * Class: com_oracle_jnitest_GetArrayRegionXorOpImpl + * Method: xor + * Signature: ([BI[BII)V + */ +JNIEXPORT void JNICALL Java_org_openjdk_bench_java_lang_foreign_xor_GetArrayRegionXorOpImpl_xor + (JNIEnv *env, jobject obj, jbyteArray src, jint sOff, jbyteArray dst, jint dOff, jint len) { + jbyte *sbuf = NULL; + jbyte *dbuf = NULL; + + sbuf = malloc(len); + dbuf = malloc(len); + + (*env)->GetByteArrayRegion(env, src, sOff, len, sbuf); + (*env)->GetByteArrayRegion(env, dst, dOff, len, dbuf); + xor_op(sbuf, dbuf, len); + (*env)->SetByteArrayRegion(env, dst, dOff, len, dbuf); + + free(dbuf); + free(sbuf); +} + +JNIEXPORT void JNICALL Java_org_openjdk_bench_java_lang_foreign_xor_GetArrayUnsafeXorOpImpl_xorOp + (JNIEnv *env, jobject obj, jlong src, jlong dst, jint len) { + jbyte *sbuf = (jbyte*)(void*)src; + jbyte *dbuf = (jbyte*)(void*)dst; + xor_op(sbuf, dbuf, len); +}