8287325: AArch64: fix virtual threads with -XX:UseBranchProtection=pac-ret

Co-authored-by: Nick Gasson <ngasson@openjdk.org>
Reviewed-by: aph, dlong
This commit is contained in:
Hao Sun 2023-09-25 05:38:51 +00:00
parent f0ff001dd7
commit 481cfc7985
23 changed files with 204 additions and 142 deletions

@ -1721,8 +1721,8 @@ void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
st->print("# stack bang size=%d\n\t", framesize);
if (VM_Version::use_rop_protection()) {
st->print("ldr zr, [lr]\n\t");
st->print("pacia lr, rfp\n\t");
st->print("ldr zr, [lr]\n\t");
st->print("paciaz\n\t");
}
if (framesize < ((1 << 9) + 2 * wordSize)) {
st->print("sub sp, sp, #%d\n\t", framesize);
@ -1851,8 +1851,8 @@ void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
}
if (VM_Version::use_rop_protection()) {
st->print("autia lr, rfp\n\t");
st->print("ldr zr, [lr]\n\t");
st->print("autiaz\n\t");
st->print("ldr zr, [lr]\n\t");
}
if (do_polling() && C->is_method_compilation()) {

@ -385,7 +385,7 @@ OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
// load issuing PC (the return address for this stub) into r3
__ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
__ authenticate_return_address(exception_pc, rscratch1);
__ authenticate_return_address(exception_pc);
// make sure that the vm_results are cleared (may be unnecessary)
__ str(zr, Address(rthread, JavaThread::vm_result_offset()));
@ -434,7 +434,7 @@ OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
__ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
// patch throwing pc into return address (has bci & oop map)
__ protect_return_address(exception_pc, rscratch1);
__ protect_return_address(exception_pc);
__ str(exception_pc, Address(rfp, 1*BytesPerWord));
// compute the exception handler.
@ -450,7 +450,7 @@ OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
__ invalidate_registers(false, true, true, true, true, true);
// patch the return address, this stub will directly return to the exception handler
__ protect_return_address(r0, rscratch1);
__ protect_return_address(r0);
__ str(r0, Address(rfp, 1*BytesPerWord));
switch (id) {

@ -62,7 +62,7 @@ inline frame FreezeBase::sender(const frame& f) {
intptr_t** link_addr = link_address<FKind>(f);
intptr_t* sender_sp = (intptr_t*)(link_addr + frame::sender_sp_offset); // f.unextended_sp() + (fsize/wordSize); //
address sender_pc = (address) *(sender_sp-1);
address sender_pc = ContinuationHelper::return_address_at(sender_sp - 1);
assert(sender_sp != f.sp(), "must have changed");
int slot = 0;

@ -68,6 +68,17 @@ inline void ContinuationHelper::push_pd(const frame& f) {
*(intptr_t**)(f.sp() - frame::sender_sp_offset) = f.fp();
}
#define CPU_OVERRIDES_RETURN_ADDRESS_ACCESSORS
inline address ContinuationHelper::return_address_at(intptr_t* sp) {
return pauth_strip_verifiable(*(address*)sp);
}
inline void ContinuationHelper::patch_return_address_at(intptr_t* sp,
address pc) {
*(address*)sp = pauth_sign_return_address(pc);
}
inline void ContinuationHelper::set_anchor_to_entry_pd(JavaFrameAnchor* anchor, ContinuationEntry* entry) {
anchor->set_last_Java_fp(entry->entry_fp());
}
@ -80,7 +91,8 @@ inline void ContinuationHelper::set_anchor_pd(JavaFrameAnchor* anchor, intptr_t*
inline bool ContinuationHelper::Frame::assert_frame_laid_out(frame f) {
intptr_t* sp = f.sp();
address pc = *(address*)(sp - frame::sender_sp_ret_address_offset());
address pc = ContinuationHelper::return_address_at(
sp - frame::sender_sp_ret_address_offset());
intptr_t* fp = *(intptr_t**)(sp - frame::sender_sp_offset);
assert(f.raw_pc() == pc, "f.ra_pc: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.raw_pc()), p2i(pc));
assert(f.fp() == fp, "f.fp: " INTPTR_FORMAT " actual: " INTPTR_FORMAT, p2i(f.fp()), p2i(fp));
@ -108,13 +120,14 @@ inline void ContinuationHelper::InterpretedFrame::patch_sender_sp(frame& f, cons
}
inline address ContinuationHelper::Frame::real_pc(const frame& f) {
// Always used in assertions. Just strip it.
address* pc_addr = &(((address*) f.sp())[-1]);
return *pc_addr;
return pauth_strip_pointer(*pc_addr);
}
inline void ContinuationHelper::Frame::patch_pc(const frame& f, address pc) {
address* pc_addr = &(((address*) f.sp())[-1]);
*pc_addr = pc;
*pc_addr = pauth_sign_return_address(pc);
}
inline intptr_t* ContinuationHelper::InterpretedFrame::frame_top(const frame& f, InterpreterOopMap* mask) { // inclusive; this will be copied with the frame

@ -139,8 +139,7 @@ bool frame::safe_for_sender(JavaThread *thread) {
sender_sp = (intptr_t*) addr_at(sender_sp_offset);
sender_unextended_sp = (intptr_t*) this->fp()[interpreter_frame_sender_sp_offset];
saved_fp = (intptr_t*) this->fp()[link_offset];
sender_pc = pauth_strip_verifiable((address) this->fp()[return_addr_offset], (address)saved_fp);
sender_pc = pauth_strip_verifiable((address) this->fp()[return_addr_offset]);
} else {
// must be some sort of compiled/runtime frame
// fp does not have to be safe (although it could be check for c1?)
@ -158,7 +157,9 @@ bool frame::safe_for_sender(JavaThread *thread) {
sender_unextended_sp = sender_sp;
// Note: frame::sender_sp_offset is only valid for compiled frame
saved_fp = (intptr_t*) *(sender_sp - frame::sender_sp_offset);
sender_pc = pauth_strip_verifiable((address) *(sender_sp-1), (address)saved_fp);
// Note: PAC authentication may fail in case broken frame is passed in.
// Just strip it for now.
sender_pc = pauth_strip_pointer((address) *(sender_sp - 1));
}
if (Continuation::is_return_barrier_entry(sender_pc)) {
@ -276,9 +277,8 @@ bool frame::safe_for_sender(JavaThread *thread) {
void frame::patch_pc(Thread* thread, address pc) {
assert(_cb == CodeCache::find_blob(pc), "unexpected pc");
address* pc_addr = &(((address*) sp())[-1]);
address signing_sp = (((address*) sp())[-2]);
address signed_pc = pauth_sign_return_address(pc, (address)signing_sp);
address pc_old = pauth_strip_verifiable(*pc_addr, (address)signing_sp);
address signed_pc = pauth_sign_return_address(pc);
address pc_old = pauth_strip_verifiable(*pc_addr);
if (TracePcPatching) {
tty->print("patch_pc at address " INTPTR_FORMAT " [" INTPTR_FORMAT " -> " INTPTR_FORMAT "]",
@ -476,8 +476,9 @@ frame frame::sender_for_interpreter_frame(RegisterMap* map) const {
}
#endif // COMPILER2_OR_JVMCI
// For ROP protection, Interpreter will have signed the sender_pc, but there is no requirement to authenticate it here.
address sender_pc = pauth_strip_verifiable(sender_pc_maybe_signed(), (address)link());
// For ROP protection, Interpreter will have signed the sender_pc,
// but there is no requirement to authenticate it here.
address sender_pc = pauth_strip_verifiable(sender_pc_maybe_signed());
if (Continuation::is_return_barrier_entry(sender_pc)) {
if (map->walk_cont()) { // about to walk into an h-stack

@ -151,7 +151,10 @@ inline frame::frame(intptr_t* sp, intptr_t* unextended_sp, intptr_t* fp, address
setup(pc);
}
inline frame::frame(intptr_t* sp) : frame(sp, sp, *(intptr_t**)(sp - frame::sender_sp_offset), *(address*)(sp - 1)) {}
inline frame::frame(intptr_t* sp)
: frame(sp, sp,
*(intptr_t**)(sp - frame::sender_sp_offset),
pauth_strip_verifiable(*(address*)(sp - 1))) {}
inline frame::frame(intptr_t* sp, intptr_t* fp) {
intptr_t a = intptr_t(sp);
@ -417,9 +420,10 @@ inline frame frame::sender_for_compiled_frame(RegisterMap* map) const {
: sender_sp();
assert(!_sp_is_trusted || l_sender_sp == real_fp(), "");
// the return_address is always the word on the stack
// For ROP protection, C1/C2 will have signed the sender_pc, but there is no requirement to authenticate it here.
address sender_pc = pauth_strip_verifiable((address) *(l_sender_sp-1), (address) *(l_sender_sp-2));
// The return_address is always the word on the stack.
// For ROP protection, C1/C2 will have signed the sender_pc,
// but there is no requirement to authenticate it here.
address sender_pc = pauth_strip_verifiable((address) *(l_sender_sp - 1));
intptr_t** saved_fp_addr = (intptr_t**) (l_sender_sp - frame::sender_sp_offset);

@ -6067,51 +6067,43 @@ void MacroAssembler::leave() {
// For more details on PAC see pauth_aarch64.hpp.
// Sign the LR. Use during construction of a stack frame, before storing the LR to memory.
// Uses the FP as the modifier.
// Uses value zero as the modifier.
//
void MacroAssembler::protect_return_address() {
if (VM_Version::use_rop_protection()) {
check_return_address();
// The standard convention for C code is to use paciasp, which uses SP as the modifier. This
// works because in C code, FP and SP match on function entry. In the JDK, SP and FP may not
// match, so instead explicitly use the FP.
pacia(lr, rfp);
paciaz();
}
}
// Sign the return value in the given register. Use before updating the LR in the existing stack
// frame for the current function.
// Uses the FP from the start of the function as the modifier - which is stored at the address of
// the current FP.
// Uses value zero as the modifier.
//
void MacroAssembler::protect_return_address(Register return_reg, Register temp_reg) {
void MacroAssembler::protect_return_address(Register return_reg) {
if (VM_Version::use_rop_protection()) {
assert(PreserveFramePointer, "PreserveFramePointer must be set for ROP protection");
check_return_address(return_reg);
ldr(temp_reg, Address(rfp));
pacia(return_reg, temp_reg);
paciza(return_reg);
}
}
// Authenticate the LR. Use before function return, after restoring FP and loading LR from memory.
// Uses value zero as the modifier.
//
void MacroAssembler::authenticate_return_address(Register return_reg) {
void MacroAssembler::authenticate_return_address() {
if (VM_Version::use_rop_protection()) {
autia(return_reg, rfp);
check_return_address(return_reg);
autiaz();
check_return_address();
}
}
// Authenticate the return value in the given register. Use before updating the LR in the existing
// stack frame for the current function.
// Uses the FP from the start of the function as the modifier - which is stored at the address of
// the current FP.
// Uses value zero as the modifier.
//
void MacroAssembler::authenticate_return_address(Register return_reg, Register temp_reg) {
void MacroAssembler::authenticate_return_address(Register return_reg) {
if (VM_Version::use_rop_protection()) {
assert(PreserveFramePointer, "PreserveFramePointer must be set for ROP protection");
ldr(temp_reg, Address(rfp));
autia(return_reg, temp_reg);
autiza(return_reg);
check_return_address(return_reg);
}
}

@ -717,9 +717,9 @@ public:
// ROP Protection
void protect_return_address();
void protect_return_address(Register return_reg, Register temp_reg);
void authenticate_return_address(Register return_reg = lr);
void authenticate_return_address(Register return_reg, Register temp_reg);
void protect_return_address(Register return_reg);
void authenticate_return_address();
void authenticate_return_address(Register return_reg);
void strip_return_address();
void check_return_address(Register return_reg=lr) PRODUCT_RETURN;

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2022, Arm Limited. All rights reserved.
* Copyright (c) 2021, 2023, Arm Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -77,9 +77,9 @@ inline bool pauth_ptr_is_raw(address ptr) {
// Strip a return value (same as pauth_strip_pointer). When debug is enabled then authenticate
// instead.
//
inline address pauth_strip_verifiable(address ret_addr, address modifier) {
inline address pauth_strip_verifiable(address ret_addr) {
if (VM_Version::use_rop_protection()) {
DEBUG_ONLY(ret_addr = pauth_authenticate_return_address(ret_addr, modifier);)
DEBUG_ONLY(ret_addr = pauth_authenticate_return_address(ret_addr);)
NOT_DEBUG(ret_addr = pauth_strip_pointer(ret_addr));
}
return ret_addr;

@ -390,7 +390,7 @@ static void patch_callers_callsite(MacroAssembler *masm) {
__ mov(c_rarg0, rmethod);
__ mov(c_rarg1, lr);
__ authenticate_return_address(c_rarg1, rscratch1);
__ authenticate_return_address(c_rarg1);
__ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
__ blr(rscratch1);
@ -1167,6 +1167,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
continuation_enter_cleanup(masm);
__ ldr(c_rarg1, Address(rfp, wordSize)); // return address
__ authenticate_return_address(c_rarg1);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
// see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc
@ -2332,7 +2333,7 @@ void SharedRuntime::generate_deopt_blob() {
// load throwing pc from JavaThread and patch it as the return address
// of the current frame. Then clear the field in JavaThread
__ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
__ protect_return_address(r3, rscratch1);
__ protect_return_address(r3);
__ str(r3, Address(rfp, wordSize));
__ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
@ -2438,9 +2439,7 @@ void SharedRuntime::generate_deopt_blob() {
__ ldrw(r2, Address(r5, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset()));
__ sub(r2, r2, 2 * wordSize);
__ add(sp, sp, r2);
__ ldp(rfp, lr, __ post(sp, 2 * wordSize));
__ authenticate_return_address();
// LR should now be the return address to the caller (3)
__ ldp(rfp, zr, __ post(sp, 2 * wordSize));
#ifdef ASSERT
// Compilers generate code that bang the stack by as much as the
@ -2655,9 +2654,7 @@ void SharedRuntime::generate_uncommon_trap_blob() {
size_of_deoptimized_frame_offset()));
__ sub(r2, r2, 2 * wordSize);
__ add(sp, sp, r2);
__ ldp(rfp, lr, __ post(sp, 2 * wordSize));
__ authenticate_return_address();
// LR should now be the return address to the caller (3) frame
__ ldp(rfp, zr, __ post(sp, 2 * wordSize));
#ifdef ASSERT
// Compilers generate code that bang the stack by as much as the
@ -2803,7 +2800,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
// it later to determine if someone changed the return address for
// us!
__ ldr(r20, Address(rthread, JavaThread::saved_exception_pc_offset()));
__ protect_return_address(r20, rscratch1);
__ protect_return_address(r20);
__ str(r20, Address(rfp, wordSize));
}
@ -2844,7 +2841,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
__ ldr(rscratch1, Address(rfp, wordSize));
__ cmp(r20, rscratch1);
__ br(Assembler::NE, no_adjust);
__ authenticate_return_address(r20, rscratch1);
__ authenticate_return_address(r20);
#ifdef ASSERT
// Verify the correct encoding of the poll we're about to skip.
@ -2859,7 +2856,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
#endif
// Adjust return pc forward to step over the safepoint poll instruction
__ add(r20, r20, NativeInstruction::instruction_size);
__ protect_return_address(r20, rscratch1);
__ protect_return_address(r20);
__ str(r20, Address(rfp, wordSize));
}

@ -26,6 +26,7 @@
#define CPU_AARCH64_STACKCHUNKFRAMESTREAM_AARCH64_INLINE_HPP
#include "interpreter/oopMapCache.hpp"
#include "pauth_aarch64.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/registerMap.hpp"
@ -52,7 +53,8 @@ inline frame StackChunkFrameStream<frame_kind>::to_frame() const {
template <ChunkFrames frame_kind>
inline address StackChunkFrameStream<frame_kind>::get_pc() const {
assert(!is_done(), "");
return *(address*)(_sp - 1);
// Just strip it for frames on the heap.
return pauth_strip_pointer(*(address*)(_sp - 1));
}
template <ChunkFrames frame_kind>

@ -7009,8 +7009,10 @@ class StubGenerator: public StubCodeGenerator {
if (return_barrier_exception) {
__ ldr(c_rarg1, Address(rfp, wordSize)); // return address
__ authenticate_return_address(c_rarg1);
__ verify_oop(r0);
__ mov(r19, r0); // save return value contaning the exception oop in callee-saved R19
// save return value containing the exception oop in callee-saved R19
__ mov(r19, r0);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
@ -7020,7 +7022,7 @@ class StubGenerator: public StubCodeGenerator {
// see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc
__ mov(r1, r0); // the exception handler
__ mov(r0, r19); // restore return value contaning the exception oop
__ mov(r0, r19); // restore return value containing the exception oop
__ verify_oop(r0);
__ leave();

@ -1833,7 +1833,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
Label caller_not_deoptimized;
__ ldr(c_rarg1, Address(rfp, frame::return_addr_offset * wordSize));
// This is a return address, so requires authenticating for PAC.
__ authenticate_return_address(c_rarg1, rscratch1);
__ authenticate_return_address(c_rarg1);
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
InterpreterRuntime::interpreter_contains), c_rarg1);
__ cbnz(r0, caller_not_deoptimized);

@ -445,16 +445,12 @@ void VM_Version::initialize() {
strcmp(UseBranchProtection, "pac-ret") == 0) {
_rop_protection = false;
// Enable ROP-protection if
// 1) this code has been built with branch-protection,
// 2) the CPU/OS supports it, and
// 3) incompatible VMContinuations isn't enabled.
// 1) this code has been built with branch-protection and
// 2) the CPU/OS supports it
#ifdef __ARM_FEATURE_PAC_DEFAULT
if (!VM_Version::supports_paca()) {
// Disable PAC to prevent illegal instruction crashes.
warning("ROP-protection specified, but not supported on this CPU. Disabling ROP-protection.");
} else if (VMContinuations) {
// Not currently compatible with continuation freeze/thaw.
warning("ROP-protection is incompatible with VMContinuations. Disabling ROP-protection.");
} else {
_rop_protection = true;
}
@ -469,12 +465,6 @@ void VM_Version::initialize() {
// Determine the mask of address bits used for PAC. Clear bit 55 of
// the input to make it look like a user address.
_pac_mask = (uintptr_t)pauth_strip_pointer((address)~(UINT64_C(1) << 55));
// The frame pointer must be preserved for ROP protection.
if (FLAG_IS_DEFAULT(PreserveFramePointer) == false && PreserveFramePointer == false ) {
vm_exit_during_initialization(err_msg("PreserveFramePointer cannot be disabled for ROP-protection"));
}
PreserveFramePointer = true;
}
#ifdef COMPILER2

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2022, Arm Limited. All rights reserved.
* Copyright (c) 2021, 2023, Arm Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -33,12 +33,12 @@ inline address pauth_strip_pointer(address ptr) {
return ptr;
}
inline address pauth_sign_return_address(address ret_addr, address sp) {
inline address pauth_sign_return_address(address ret_addr) {
// No PAC support in BSD as of yet.
return ret_addr;
}
inline address pauth_authenticate_return_address(address ret_addr, address sp) {
inline address pauth_authenticate_return_address(address ret_addr) {
// No PAC support in BSD as of yet.
return ret_addr;
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2022, Arm Limited. All rights reserved.
* Copyright (c) 2021, 2023, Arm Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -30,15 +30,14 @@
inline bool pauth_ptr_is_raw(address ptr);
// Use only the PAC instructions in the NOP space. This ensures the binaries work on systems
// without PAC. Write these instructions using their alternate "hint" instructions to ensure older
// compilers can still be used.
#define XPACLRI "hint #0x7;"
#define PACIA1716 "hint #0x8;"
#define AUTIA1716 "hint #0xc;"
// Write these instructions using their alternate "hint" instructions to
// ensure older compilers can still be used.
#define XPACLRI "hint #0x7;"
#define PACIAZ "hint #0x18;"
#define AUTIAZ "hint #0x1c;"
// Strip an address. Use with caution - only if there is no guaranteed way of authenticating the
// value.
// Strip an address. Use with caution -
// only if there is no guaranteed way of authenticating the value.
//
inline address pauth_strip_pointer(address ptr) {
register address result __asm__("x30") = ptr;
@ -46,36 +45,35 @@ inline address pauth_strip_pointer(address ptr) {
return result;
}
// Sign a return value, using the given modifier.
// Sign a return value, using value zero as the modifier.
//
inline address pauth_sign_return_address(address ret_addr, address sp) {
inline address pauth_sign_return_address(address ret_addr) {
if (VM_Version::use_rop_protection()) {
// A pointer cannot be double signed.
guarantee(pauth_ptr_is_raw(ret_addr), "Return address is already signed");
register address r17 __asm("r17") = ret_addr;
register address r16 __asm("r16") = sp;
asm (PACIA1716 : "+r"(r17) : "r"(r16));
ret_addr = r17;
register address reg30 __asm__("x30") = ret_addr;
asm (PACIAZ : "+r"(reg30));
ret_addr = reg30;
}
return ret_addr;
}
// Authenticate a return value, using the given modifier.
// Authenticate a return value, using value zero as the modifier.
//
inline address pauth_authenticate_return_address(address ret_addr, address sp) {
inline address pauth_authenticate_return_address(address ret_addr) {
if (VM_Version::use_rop_protection()) {
register address r17 __asm("r17") = ret_addr;
register address r16 __asm("r16") = sp;
asm (AUTIA1716 : "+r"(r17) : "r"(r16));
ret_addr = r17;
register address reg30 __asm__("x30") = ret_addr;
asm (AUTIAZ : "+r"(reg30));
ret_addr = reg30;
// Ensure that the pointer authenticated.
guarantee(pauth_ptr_is_raw(ret_addr), "Return address did not authenticate");
guarantee(pauth_ptr_is_raw(ret_addr),
"Return address did not authenticate");
}
return ret_addr;
}
#undef XPACLRI
#undef PACIA1716
#undef AUTIA1716
#undef PACIAZ
#undef AUTIAZ
#endif // OS_CPU_LINUX_AARCH64_PAUTH_LINUX_AARCH64_INLINE_HPP

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2022, Arm Limited. All rights reserved.
* Copyright (c) 2021, 2023, Arm Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -33,12 +33,12 @@ inline address pauth_strip_pointer(address ptr) {
return ptr;
}
inline address pauth_sign_return_address(address ret_addr, address sp) {
inline address pauth_sign_return_address(address ret_addr) {
// No PAC support in windows as of yet.
return ret_addr;
}
inline address pauth_authenticate_return_address(address ret_addr, address sp) {
inline address pauth_authenticate_return_address(address ret_addr) {
// No PAC support in windows as of yet.
return ret_addr;
}

@ -1563,10 +1563,7 @@ address OptoRuntime::handle_exception_C(JavaThread* current) {
//
address OptoRuntime::rethrow_C(oopDesc* exception, JavaThread* thread, address ret_pc) {
// ret_pc will have been loaded from the stack, so for AArch64 will be signed.
// This needs authenticating, but to do that here requires the fp of the previous frame.
// A better way of doing it would be authenticate in the caller by adding a
// AuthPAuthNode and using it in GraphKit::gen_stub. For now, just strip it.
AARCH64_PORT_ONLY(ret_pc = pauth_strip_pointer(ret_pc));
AARCH64_PORT_ONLY(ret_pc = pauth_strip_verifiable(ret_pc));
#ifndef PRODUCT
SharedRuntime::_rethrow_ctr++; // count rethrows

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -28,6 +28,7 @@
#include "oops/method.inline.hpp"
#include "runtime/continuation.hpp"
#include "runtime/continuationEntry.inline.hpp"
#include "runtime/continuationHelper.inline.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/stackFrameStream.inline.hpp"
@ -134,7 +135,8 @@ bool ContinuationEntry::assert_entry_frame_laid_out(JavaThread* thread) {
assert(sp != nullptr, "");
assert(sp <= entry->entry_sp(), "");
address pc = *(address*)(sp - frame::sender_sp_ret_address_offset());
address pc = ContinuationHelper::return_address_at(
sp - frame::sender_sp_ret_address_offset());
if (pc != StubRoutines::cont_returnBarrier()) {
CodeBlob* cb = pc != nullptr ? CodeCache::find_blob(pc) : nullptr;

@ -299,7 +299,8 @@ inline void clear_anchor(JavaThread* thread) {
}
static void set_anchor(JavaThread* thread, intptr_t* sp) {
address pc = *(address*)(sp - frame::sender_sp_ret_address_offset());
address pc = ContinuationHelper::return_address_at(
sp - frame::sender_sp_ret_address_offset());
assert(pc != nullptr, "");
JavaFrameAnchor* anchor = thread->frame_anchor();
@ -592,7 +593,14 @@ void FreezeBase::freeze_fast_existing_chunk() {
if (chunk->sp() < chunk->stack_size()) { // we are copying into a non-empty chunk
DEBUG_ONLY(_empty = false;)
assert(chunk->sp() < (chunk->stack_size() - chunk->argsize()), "");
assert(*(address*)(chunk->sp_address() - frame::sender_sp_ret_address_offset()) == chunk->pc(), "");
#ifdef ASSERT
{
intptr_t* retaddr_slot = (chunk->sp_address()
- frame::sender_sp_ret_address_offset());
assert(ContinuationHelper::return_address_at(retaddr_slot) == chunk->pc(),
"unexpected saved return address");
}
#endif
// the chunk's sp before the freeze, adjusted to point beyond the stack-passed arguments in the topmost frame
// we overlap; we'll overwrite the chunk's top frame's callee arguments
@ -606,8 +614,15 @@ void FreezeBase::freeze_fast_existing_chunk() {
assert(bottom_sp == _bottom_address, "");
// Because the chunk isn't empty, we know there's a caller in the chunk, therefore the bottom-most frame
// should have a return barrier (installed back when we thawed it).
assert(*(address*)(bottom_sp-frame::sender_sp_ret_address_offset()) == StubRoutines::cont_returnBarrier(),
"should be the continuation return barrier");
#ifdef ASSERT
{
intptr_t* retaddr_slot = (bottom_sp
- frame::sender_sp_ret_address_offset());
assert(ContinuationHelper::return_address_at(retaddr_slot)
== StubRoutines::cont_returnBarrier(),
"should be the continuation return barrier");
}
#endif
// We copy the fp from the chunk back to the stack because it contains some caller data,
// including, possibly, an oop that might have gone stale since we thawed.
patch_stack_pd(bottom_sp, chunk->sp_address());
@ -675,7 +690,14 @@ void FreezeBase::freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_J
assert(!(_fast_freeze_size > 0) || _orig_chunk_sp - (chunk->start_address() + chunk_new_sp) == _fast_freeze_size, "");
intptr_t* chunk_top = chunk->start_address() + chunk_new_sp;
assert(_empty || *(address*)(_orig_chunk_sp - frame::sender_sp_ret_address_offset()) == chunk->pc(), "");
#ifdef ASSERT
if (!_empty) {
intptr_t* retaddr_slot = (_orig_chunk_sp
- frame::sender_sp_ret_address_offset());
assert(ContinuationHelper::return_address_at(retaddr_slot) == chunk->pc(),
"unexpected saved return address");
}
#endif
log_develop_trace(continuations)("freeze_fast start: " INTPTR_FORMAT " sp: %d chunk_top: " INTPTR_FORMAT,
p2i(chunk->start_address()), chunk_new_sp, p2i(chunk_top));
@ -684,15 +706,27 @@ void FreezeBase::freeze_fast_copy(stackChunkOop chunk, int chunk_start_sp CONT_J
copy_to_chunk(from, to, cont_size() + frame::metadata_words_at_bottom);
// Because we're not patched yet, the chunk is now in a bad state
// patch return pc of the bottom-most frozen frame (now in the chunk) with the actual caller's return address
intptr_t* chunk_bottom_sp = chunk_top + cont_size() - _cont.argsize() - frame::metadata_words_at_top;
assert(_empty || *(address*)(chunk_bottom_sp-frame::sender_sp_ret_address_offset()) == StubRoutines::cont_returnBarrier(), "");
*(address*)(chunk_bottom_sp - frame::sender_sp_ret_address_offset()) = chunk->pc();
// patch return pc of the bottom-most frozen frame (now in the chunk)
// with the actual caller's return address
intptr_t* chunk_bottom_retaddr_slot = (chunk_top + cont_size()
- _cont.argsize()
- frame::metadata_words_at_top
- frame::sender_sp_ret_address_offset());
#ifdef ASSERT
if (!_empty) {
assert(ContinuationHelper::return_address_at(chunk_bottom_retaddr_slot)
== StubRoutines::cont_returnBarrier(),
"should be the continuation return barrier");
}
#endif
ContinuationHelper::patch_return_address_at(chunk_bottom_retaddr_slot,
chunk->pc());
// We're always writing to a young chunk, so the GC can't see it until the next safepoint.
chunk->set_sp(chunk_new_sp);
// set chunk->pc to the return address of the topmost frame in the chunk
chunk->set_pc(*(address*)(_cont_stack_top - frame::sender_sp_ret_address_offset()));
chunk->set_pc(ContinuationHelper::return_address_at(
_cont_stack_top - frame::sender_sp_ret_address_offset()));
_cont.write();
@ -1846,7 +1880,15 @@ inline void ThawBase::clear_chunk(stackChunkOop chunk) {
chunk->set_max_thawing_size(chunk->max_thawing_size() - frame_size);
// We set chunk->pc to the return pc into the next frame
chunk->set_pc(f.pc());
assert(f.pc() == *(address*)(chunk_sp + frame_size - frame::sender_sp_ret_address_offset()), "unexpected pc");
#ifdef ASSERT
{
intptr_t* retaddr_slot = (chunk_sp
+ frame_size
- frame::sender_sp_ret_address_offset());
assert(f.pc() == ContinuationHelper::return_address_at(retaddr_slot),
"unexpected pc");
}
#endif
}
assert(empty == chunk->is_empty(), "");
// returns the size required to store the frame on stack, and because it is a
@ -1866,7 +1908,9 @@ void ThawBase::patch_return(intptr_t* sp, bool is_last) {
log_develop_trace(continuations)("thaw_fast patching -- sp: " INTPTR_FORMAT, p2i(sp));
address pc = !is_last ? StubRoutines::cont_returnBarrier() : _cont.entryPC();
*(address*)(sp - frame::sender_sp_ret_address_offset()) = pc;
ContinuationHelper::patch_return_address_at(
sp - frame::sender_sp_ret_address_offset(),
pc);
}
template <typename ConfigT>
@ -2403,7 +2447,6 @@ static inline intptr_t* thaw_internal(JavaThread* thread, const Continuation::th
#ifdef ASSERT
intptr_t* sp0 = sp;
address pc0 = *(address*)(sp - frame::sender_sp_ret_address_offset());
set_anchor(thread, sp0);
log_frames(thread);
if (LoomVerifyAfterThaw) {

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -44,6 +44,9 @@ public:
static inline void push_pd(const frame& f);
static inline address return_address_at(intptr_t* sp);
static inline void patch_return_address_at(intptr_t* sp, address pc);
static inline int frame_align_words(int size);
static inline intptr_t* frame_align_pointer(intptr_t* sp);
@ -68,7 +71,7 @@ public:
static inline address real_pc(const frame& f);
static inline void patch_pc(const frame& f, address pc);
static address* return_pc_address(const frame& f);
static address return_pc(const frame& f) { return *return_pc_address(f); }
static address return_pc(const frame& f);
static bool is_stub(CodeBlob* cb);
#ifdef ASSERT

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -37,6 +37,17 @@
#include CPU_HEADER_INLINE(continuationHelper)
#ifndef CPU_OVERRIDES_RETURN_ADDRESS_ACCESSORS
inline address ContinuationHelper::return_address_at(intptr_t* sp) {
return *(address*)sp;
}
inline void ContinuationHelper::patch_return_address_at(intptr_t* sp,
address pc) {
*(address*)sp = pc;
}
#endif // !CPU_OVERRIDES_RETURN_ADDRESS_ACCESSORS
inline bool ContinuationHelper::NonInterpretedUnknownFrame::is_instance(const frame& f) {
return !f.is_interpreted_frame();
}
@ -49,6 +60,10 @@ inline Method* ContinuationHelper::Frame::frame_method(const frame& f) {
return f.is_interpreted_frame() ? f.interpreter_frame_method() : f.cb()->as_compiled_method()->method();
}
inline address ContinuationHelper::Frame::return_pc(const frame& f) {
return return_address_at((intptr_t *)return_pc_address(f));
}
#ifdef ASSERT
inline intptr_t* ContinuationHelper::Frame::frame_top(const frame &f) {
if (f.is_interpreted_frame()) {
@ -75,7 +90,7 @@ inline bool ContinuationHelper::InterpretedFrame::is_instance(const frame& f) {
}
inline address ContinuationHelper::InterpretedFrame::return_pc(const frame& f) {
return *return_pc_address(f);
return return_address_at((intptr_t *)return_pc_address(f));
}
inline int ContinuationHelper::InterpretedFrame::size(const frame&f) {

@ -1,6 +1,6 @@
/*
* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Arm Limited. All rights reserved.
* Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2023, Arm Limited. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -255,10 +255,11 @@ public class AArch64TestAssembler extends TestAssembler {
// Must be patchable by NativeJump::patch_verified_entry
emitNop();
if (config.ropProtection) {
code.emitInt(0xdac103be); // pacia x30, x29
code.emitInt(0xf94003df); // ldr xzr, [x30]
code.emitInt(0xd503231f); // paciaz
}
code.emitInt(0xa9be7bfd); // stp x29, x30, [sp, #-32]!
code.emitInt(0x910003fd); // mov x29, sp
code.emitInt(0xa9bf7bfd); // stp x29, x30, [sp, #-16]!
code.emitInt(0x910003fd); // mov x29, sp
setDeoptRescueSlot(newStackSlot(AArch64Kind.QWORD));
}
@ -468,23 +469,25 @@ public class AArch64TestAssembler extends TestAssembler {
@Override
public void emitIntRet(Register a) {
emitMov(AArch64.r0, a);
code.emitInt(0x910003bf); // mov sp, x29
code.emitInt(0xa8c27bfd); // ldp x29, x30, [sp], #32
code.emitInt(0x910003bf); // mov sp, x29
code.emitInt(0xa8c17bfd); // ldp x29, x30, [sp], #16
if (config.ropProtection) {
code.emitInt(0xdac113be); // autia x30, x29
code.emitInt(0xd503239f); // autiaz
code.emitInt(0xf94003df); // ldr xzr, [x30]
}
code.emitInt(0xd65f03c0); // ret
code.emitInt(0xd65f03c0); // ret
}
@Override
public void emitFloatRet(Register a) {
assert a == AArch64.v0 : "Unimplemented move " + a;
code.emitInt(0x910003bf); // mov sp, x29
code.emitInt(0xa8c27bfd); // ldp x29, x30, [sp], #32
code.emitInt(0x910003bf); // mov sp, x29
code.emitInt(0xa8c17bfd); // ldp x29, x30, [sp], #16
if (config.ropProtection) {
code.emitInt(0xdac113be); // autia x30, x29
code.emitInt(0xd503239f); // autiaz
code.emitInt(0xf94003df); // ldr xzr, [x30]
}
code.emitInt(0xd65f03c0); // ret
code.emitInt(0xd65f03c0); // ret
}
@Override