8319947: Recursive lightweight locking: s390x implementation
Reviewed-by: aboldtch, lucy
This commit is contained in:
Amit Kumar
parent
c47a0e005e
commit
d457609f70
@ -67,9 +67,6 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox
|
||||
|
||||
verify_oop(Roop, FILE_AND_LINE);
|
||||
|
||||
// Load object header.
|
||||
z_lg(Rmark, Address(Roop, hdr_offset));
|
||||
|
||||
// Save object being locked into the BasicObjectLock...
|
||||
z_stg(Roop, Address(Rbox, BasicObjectLock::obj_offset()));
|
||||
|
||||
@ -85,6 +82,10 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox
|
||||
lightweight_lock(Roop, Rmark, tmp, slow_case);
|
||||
} else if (LockingMode == LM_LEGACY) {
|
||||
NearLabel done;
|
||||
|
||||
// Load object header.
|
||||
z_lg(Rmark, Address(Roop, hdr_offset));
|
||||
|
||||
// and mark it as unlocked.
|
||||
z_oill(Rmark, markWord::unlocked_value);
|
||||
// Save unlocked object header into the displaced header location on the stack.
|
||||
@ -141,12 +142,7 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb
|
||||
verify_oop(Roop, FILE_AND_LINE);
|
||||
|
||||
if (LockingMode == LM_LIGHTWEIGHT) {
|
||||
const Register tmp = Z_R1_scratch;
|
||||
z_lg(Rmark, Address(Roop, hdr_offset));
|
||||
z_lgr(tmp, Rmark);
|
||||
z_nill(tmp, markWord::monitor_value);
|
||||
branch_optimized(Assembler::bcondNotZero, slow_case);
|
||||
lightweight_unlock(Roop, Rmark, tmp, slow_case);
|
||||
lightweight_unlock(Roop, Rmark, Z_R1_scratch, slow_case);
|
||||
} else if (LockingMode == LM_LEGACY) {
|
||||
// Test if object header is pointing to the displaced header, and if so, restore
|
||||
// the displaced header in the object. If the object header is not pointing to
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
|
||||
* Copyright (c) 2017, 2022 SAP SE. All rights reserved.
|
||||
* Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
|
||||
* Copyright (c) 2017, 2024 SAP SE. All rights reserved.
|
||||
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
||||
*
|
||||
* This code is free software; you can redistribute it and/or modify it
|
||||
@ -33,6 +33,15 @@
|
||||
#define BLOCK_COMMENT(str) block_comment(str)
|
||||
#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
|
||||
|
||||
void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2) {
|
||||
compiler_fast_lock_lightweight_object(obj, temp1, temp2);
|
||||
}
|
||||
|
||||
|
||||
void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2) {
|
||||
compiler_fast_unlock_lightweight_object(obj, temp1, temp2);
|
||||
}
|
||||
|
||||
//------------------------------------------------------
|
||||
// Special String Intrinsics. Implementation
|
||||
//------------------------------------------------------
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
|
||||
* Copyright (c) 2017, 2022 SAP SE. All rights reserved.
|
||||
* Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
|
||||
* Copyright (c) 2017, 2024 SAP SE. All rights reserved.
|
||||
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
|
||||
*
|
||||
* This code is free software; you can redistribute it and/or modify it
|
||||
@ -29,6 +29,10 @@
|
||||
// C2_MacroAssembler contains high-level macros for C2
|
||||
|
||||
public:
|
||||
// Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in s390.ad file.
|
||||
void fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2);
|
||||
void fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2);
|
||||
|
||||
//-------------------------------------------
|
||||
// Special String Intrinsics Implementation.
|
||||
//-------------------------------------------
|
||||
|
||||
@ -1005,9 +1005,6 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
|
||||
|
||||
// markWord header = obj->mark().set_unlocked();
|
||||
|
||||
// Load markWord from object into header.
|
||||
z_lg(header, hdr_offset, object);
|
||||
|
||||
if (DiagnoseSyncOnValueBasedClasses != 0) {
|
||||
load_klass(tmp, object);
|
||||
testbit(Address(tmp, Klass::access_flags_offset()), exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
|
||||
@ -1015,9 +1012,12 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
|
||||
}
|
||||
|
||||
if (LockingMode == LM_LIGHTWEIGHT) {
|
||||
lightweight_lock(object, /* mark word */ header, tmp, slow_case);
|
||||
lightweight_lock(object, header, tmp, slow_case);
|
||||
} else if (LockingMode == LM_LEGACY) {
|
||||
|
||||
// Load markWord from object into header.
|
||||
z_lg(header, hdr_offset, object);
|
||||
|
||||
// Set header to be (markWord of object | UNLOCK_VALUE).
|
||||
// This will not change anything if it was unlocked before.
|
||||
z_oill(header, markWord::unlocked_value);
|
||||
@ -1153,26 +1153,8 @@ void InterpreterMacroAssembler::unlock_object(Register monitor, Register object)
|
||||
|
||||
// If we still have a lightweight lock, unlock the object and be done.
|
||||
if (LockingMode == LM_LIGHTWEIGHT) {
|
||||
// Check for non-symmetric locking. This is allowed by the spec and the interpreter
|
||||
// must handle it.
|
||||
|
||||
Register tmp = current_header;
|
||||
|
||||
// First check for lock-stack underflow.
|
||||
z_lgf(tmp, Address(Z_thread, JavaThread::lock_stack_top_offset()));
|
||||
compareU32_and_branch(tmp, (unsigned)LockStack::start_offset(), Assembler::bcondNotHigh, slow_case);
|
||||
|
||||
// Then check if the top of the lock-stack matches the unlocked object.
|
||||
z_aghi(tmp, -oopSize);
|
||||
z_lg(tmp, Address(Z_thread, tmp));
|
||||
compare64_and_branch(tmp, object, Assembler::bcondNotEqual, slow_case);
|
||||
|
||||
z_lg(header, Address(object, hdr_offset));
|
||||
z_lgr(tmp, header);
|
||||
z_nill(tmp, markWord::monitor_value);
|
||||
z_brne(slow_case);
|
||||
|
||||
lightweight_unlock(object, header, tmp, slow_case);
|
||||
lightweight_unlock(object, header, current_header, slow_case);
|
||||
|
||||
z_bru(done);
|
||||
} else {
|
||||
|
||||
@ -3190,16 +3190,20 @@ void MacroAssembler::increment_counter_eq(address counter_address, Register tmp1
|
||||
bind(l);
|
||||
}
|
||||
|
||||
// "The box" is the space on the stack where we copy the object mark.
|
||||
void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2) {
|
||||
|
||||
assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_lock_lightweight");
|
||||
assert_different_registers(oop, box, temp1, temp2);
|
||||
|
||||
Register displacedHeader = temp1;
|
||||
Register currentHeader = temp1;
|
||||
Register temp = temp2;
|
||||
Register currentHeader = temp1;
|
||||
Register temp = temp2;
|
||||
|
||||
NearLabel done, object_has_monitor;
|
||||
|
||||
const int hdr_offset = oopDesc::mark_offset_in_bytes();
|
||||
|
||||
assert_different_registers(temp1, temp2, oop, box);
|
||||
|
||||
BLOCK_COMMENT("compiler_fast_lock_object {");
|
||||
|
||||
// Load markWord from oop into mark.
|
||||
@ -3207,8 +3211,10 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis
|
||||
|
||||
if (DiagnoseSyncOnValueBasedClasses != 0) {
|
||||
load_klass(temp, oop);
|
||||
testbit(Address(temp, Klass::access_flags_offset()), exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
|
||||
z_btrue(done);
|
||||
z_l(temp, Address(temp, Klass::access_flags_offset()));
|
||||
assert((JVM_ACC_IS_VALUE_BASED_CLASS & 0xFFFF) == 0, "or change following instruction");
|
||||
z_nilh(temp, JVM_ACC_IS_VALUE_BASED_CLASS >> 16);
|
||||
z_brne(done);
|
||||
}
|
||||
|
||||
// Handle existing monitor.
|
||||
@ -3222,7 +3228,8 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis
|
||||
// From loading the markWord, we know that oop != nullptr
|
||||
z_ltgr(oop, oop);
|
||||
z_bru(done);
|
||||
} else if (LockingMode == LM_LEGACY) {
|
||||
} else {
|
||||
assert(LockingMode == LM_LEGACY, "must be");
|
||||
// Set mark to markWord | markWord::unlocked_value.
|
||||
z_oill(displacedHeader, markWord::unlocked_value);
|
||||
|
||||
@ -3251,10 +3258,6 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis
|
||||
|
||||
z_stg(currentHeader/*==0 or not 0*/, BasicLock::displaced_header_offset_in_bytes(), box);
|
||||
|
||||
z_bru(done);
|
||||
} else {
|
||||
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
|
||||
lightweight_lock(oop, displacedHeader, temp, done);
|
||||
z_bru(done);
|
||||
}
|
||||
|
||||
@ -3270,10 +3273,9 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis
|
||||
// Otherwise, register zero is filled with the current owner.
|
||||
z_lghi(zero, 0);
|
||||
z_csg(zero, Z_thread, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor_tagged);
|
||||
if (LockingMode != LM_LIGHTWEIGHT) {
|
||||
// Store a non-null value into the box.
|
||||
z_stg(box, BasicLock::displaced_header_offset_in_bytes(), box);
|
||||
}
|
||||
|
||||
// Store a non-null value into the box.
|
||||
z_stg(box, BasicLock::displaced_header_offset_in_bytes(), box);
|
||||
|
||||
z_bre(done); // acquired the lock for the first time.
|
||||
|
||||
@ -3295,14 +3297,16 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis
|
||||
}
|
||||
|
||||
void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2) {
|
||||
|
||||
assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
|
||||
assert_different_registers(oop, box, temp1, temp2);
|
||||
|
||||
Register displacedHeader = temp1;
|
||||
Register currentHeader = temp2;
|
||||
Register temp = temp1;
|
||||
Register currentHeader = temp2;
|
||||
Register temp = temp1;
|
||||
|
||||
const int hdr_offset = oopDesc::mark_offset_in_bytes();
|
||||
|
||||
assert_different_registers(temp1, temp2, oop, box);
|
||||
|
||||
Label done, object_has_monitor, not_recursive;
|
||||
|
||||
BLOCK_COMMENT("compiler_fast_unlock_object {");
|
||||
@ -3326,18 +3330,14 @@ void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Reg
|
||||
// Set NE to indicate 'failure' -> take slow-path
|
||||
z_ltgr(oop, oop);
|
||||
z_bru(done);
|
||||
} else if (LockingMode == LM_LEGACY) {
|
||||
} else {
|
||||
assert(LockingMode == LM_LEGACY, "must be");
|
||||
// Check if it is still a lightweight lock, this is true if we see
|
||||
// the stack address of the basicLock in the markWord of the object
|
||||
// copy box to currentHeader such that csg does not kill it.
|
||||
z_lgr(currentHeader, box);
|
||||
z_csg(currentHeader, displacedHeader, hdr_offset, oop);
|
||||
z_bru(done); // csg sets CR as desired.
|
||||
} else {
|
||||
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
|
||||
|
||||
lightweight_unlock(oop, currentHeader, displacedHeader, done);
|
||||
z_bru(done);
|
||||
}
|
||||
|
||||
// In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
|
||||
@ -5705,103 +5705,426 @@ SkipIfEqual::~SkipIfEqual() {
|
||||
}
|
||||
|
||||
// Implements lightweight-locking.
|
||||
// Branches to slow upon failure to lock the object.
|
||||
// Falls through upon success.
|
||||
//
|
||||
// - obj: the object to be locked, contents preserved.
|
||||
// - hdr: the header, already loaded from obj, contents destroyed.
|
||||
// - temp1, temp2: temporary registers, contents destroyed.
|
||||
// Note: make sure Z_R1 is not manipulated here when C2 compiler is in play
|
||||
void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register temp, Label& slow_case) {
|
||||
void MacroAssembler::lightweight_lock(Register obj, Register temp1, Register temp2, Label& slow) {
|
||||
|
||||
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
|
||||
assert_different_registers(obj, hdr, temp);
|
||||
assert_different_registers(obj, temp1, temp2);
|
||||
|
||||
Label push;
|
||||
const Register top = temp1;
|
||||
const Register mark = temp2;
|
||||
const int mark_offset = oopDesc::mark_offset_in_bytes();
|
||||
const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset();
|
||||
|
||||
// Preload the markWord. It is important that this is the first
|
||||
// instruction emitted as it is part of C1's null check semantics.
|
||||
z_lg(mark, Address(obj, mark_offset));
|
||||
|
||||
|
||||
// First we need to check if the lock-stack has room for pushing the object reference.
|
||||
z_lgf(temp, Address(Z_thread, JavaThread::lock_stack_top_offset()));
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
|
||||
compareU32_and_branch(temp, (unsigned)LockStack::end_offset()-1, bcondHigh, slow_case);
|
||||
compareU32_and_branch(top, (unsigned)LockStack::end_offset(), bcondNotLow, slow);
|
||||
|
||||
// attempting a lightweight_lock
|
||||
// Load (object->mark() | 1) into hdr
|
||||
z_oill(hdr, markWord::unlocked_value);
|
||||
// The underflow check is elided. The recursive check will always fail
|
||||
// when the lock stack is empty because of the _bad_oop_sentinel field.
|
||||
|
||||
z_lgr(temp, hdr);
|
||||
// Check for recursion:
|
||||
z_aghi(top, -oopSize);
|
||||
z_cg(obj, Address(Z_thread, top));
|
||||
z_bre(push);
|
||||
|
||||
// Clear lock-bits from hdr (locked state)
|
||||
z_xilf(temp, markWord::unlocked_value);
|
||||
// Check header for monitor (0b10).
|
||||
z_tmll(mark, markWord::monitor_value);
|
||||
branch_optimized(bcondNotAllZero, slow);
|
||||
|
||||
z_csg(hdr, temp, oopDesc::mark_offset_in_bytes(), obj);
|
||||
branch_optimized(Assembler::bcondNotEqual, slow_case);
|
||||
{ // Try to lock. Transition lock bits 0b01 => 0b00
|
||||
const Register locked_obj = top;
|
||||
z_oill(mark, markWord::unlocked_value);
|
||||
z_lgr(locked_obj, mark);
|
||||
// Clear lock-bits from locked_obj (locked state)
|
||||
z_xilf(locked_obj, markWord::unlocked_value);
|
||||
z_csg(mark, locked_obj, mark_offset, obj);
|
||||
branch_optimized(Assembler::bcondNotEqual, slow);
|
||||
}
|
||||
|
||||
bind(push);
|
||||
|
||||
// After successful lock, push object on lock-stack
|
||||
z_lgf(temp, Address(Z_thread, JavaThread::lock_stack_top_offset()));
|
||||
z_stg(obj, Address(Z_thread, temp));
|
||||
z_ahi(temp, oopSize);
|
||||
z_st(temp, Address(Z_thread, JavaThread::lock_stack_top_offset()));
|
||||
|
||||
// as locking was successful, set CC to EQ
|
||||
z_cr(temp, temp);
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
z_stg(obj, Address(Z_thread, top));
|
||||
z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize);
|
||||
}
|
||||
|
||||
// Implements lightweight-unlocking.
|
||||
// Branches to slow upon failure.
|
||||
// Falls through upon success.
|
||||
//
|
||||
// - obj: the object to be unlocked
|
||||
// - hdr: the (pre-loaded) header of the object, will be destroyed
|
||||
// - temp1, temp2: temporary registers, will be destroyed
|
||||
// - Z_R1_scratch: will be killed in case of Interpreter & C1 Compiler
|
||||
void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow) {
|
||||
void MacroAssembler::lightweight_unlock(Register obj, Register temp1, Register temp2, Label& slow) {
|
||||
|
||||
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
|
||||
assert_different_registers(obj, hdr, tmp);
|
||||
assert_different_registers(obj, temp1, temp2);
|
||||
|
||||
Label unlocked, push_and_slow;
|
||||
const Register mark = temp1;
|
||||
const Register top = temp2;
|
||||
const int mark_offset = oopDesc::mark_offset_in_bytes();
|
||||
const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset();
|
||||
|
||||
#ifdef ASSERT
|
||||
{
|
||||
// Check that hdr is lightweight-locked.
|
||||
Label hdr_ok;
|
||||
z_lgr(tmp, hdr);
|
||||
z_nill(tmp, markWord::lock_mask_in_place);
|
||||
z_bre(hdr_ok);
|
||||
stop("Header is not lightweight-locked");
|
||||
bind(hdr_ok);
|
||||
}
|
||||
{
|
||||
// The following checks rely on the fact that LockStack is only ever modified by
|
||||
// its owning thread, even if the lock got inflated concurrently; removal of LockStack
|
||||
// entries after inflation will happen delayed in that case.
|
||||
|
||||
// Check for lock-stack underflow.
|
||||
Label stack_ok;
|
||||
z_lgf(tmp, Address(Z_thread, JavaThread::lock_stack_top_offset()));
|
||||
compareU32_and_branch(tmp, (unsigned)LockStack::start_offset(), Assembler::bcondHigh, stack_ok);
|
||||
NearLabel stack_ok;
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
compareU32_and_branch(top, (unsigned)LockStack::start_offset(), bcondNotLow, stack_ok);
|
||||
stop("Lock-stack underflow");
|
||||
bind(stack_ok);
|
||||
}
|
||||
{
|
||||
// Check if the top of the lock-stack matches the unlocked object.
|
||||
Label tos_ok;
|
||||
z_aghi(tmp, -oopSize);
|
||||
z_lg(tmp, Address(Z_thread, tmp));
|
||||
compare64_and_branch(tmp, obj, Assembler::bcondEqual, tos_ok);
|
||||
stop("Top of lock-stack does not match the unlocked object");
|
||||
bind(tos_ok);
|
||||
}
|
||||
#endif // ASSERT
|
||||
|
||||
z_lgr(tmp, hdr);
|
||||
z_oill(tmp, markWord::unlocked_value);
|
||||
z_csg(hdr, tmp, oopDesc::mark_offset_in_bytes(), obj);
|
||||
branch_optimized(Assembler::bcondNotEqual, slow);
|
||||
// Check if obj is top of lock-stack.
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
z_aghi(top, -oopSize);
|
||||
z_cg(obj, Address(Z_thread, top));
|
||||
branch_optimized(bcondNotEqual, slow);
|
||||
|
||||
// After successful unlock, pop object from lock-stack
|
||||
// pop object from lock-stack
|
||||
#ifdef ASSERT
|
||||
z_lgf(tmp, Address(Z_thread, JavaThread::lock_stack_top_offset()));
|
||||
z_aghi(tmp, -oopSize);
|
||||
z_agr(tmp, Z_thread);
|
||||
z_xc(0, oopSize-1, tmp, 0, tmp); // wipe out lock-stack entry
|
||||
const Register temp_top = temp1; // mark is not yet loaded, but be careful
|
||||
z_agrk(temp_top, top, Z_thread);
|
||||
z_xc(0, oopSize-1, temp_top, 0, temp_top); // wipe out lock-stack entry
|
||||
#endif // ASSERT
|
||||
z_alsi(in_bytes(ls_top_offset), Z_thread, -oopSize); // pop object
|
||||
|
||||
// The underflow check is elided. The recursive check will always fail
|
||||
// when the lock stack is empty because of the _bad_oop_sentinel field.
|
||||
|
||||
// Check if recursive. (this is a check for the 2nd object on the stack)
|
||||
z_aghi(top, -oopSize);
|
||||
z_cg(obj, Address(Z_thread, top));
|
||||
branch_optimized(bcondEqual, unlocked);
|
||||
|
||||
// Not recursive. Check header for monitor (0b10).
|
||||
z_lg(mark, Address(obj, mark_offset));
|
||||
z_tmll(mark, markWord::monitor_value);
|
||||
z_brnaz(push_and_slow);
|
||||
|
||||
#ifdef ASSERT
|
||||
// Check header not unlocked (0b01).
|
||||
NearLabel not_unlocked;
|
||||
z_tmll(mark, markWord::unlocked_value);
|
||||
z_braz(not_unlocked);
|
||||
stop("lightweight_unlock already unlocked");
|
||||
bind(not_unlocked);
|
||||
#endif // ASSERT
|
||||
|
||||
{ // Try to unlock. Transition lock bits 0b00 => 0b01
|
||||
Register unlocked_obj = top;
|
||||
z_lgr(unlocked_obj, mark);
|
||||
z_oill(unlocked_obj, markWord::unlocked_value);
|
||||
z_csg(mark, unlocked_obj, mark_offset, obj);
|
||||
branch_optimized(Assembler::bcondEqual, unlocked);
|
||||
}
|
||||
|
||||
bind(push_and_slow);
|
||||
|
||||
// Restore lock-stack and handle the unlock in runtime.
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
DEBUG_ONLY(z_stg(obj, Address(Z_thread, top));)
|
||||
z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize);
|
||||
// set CC to NE
|
||||
z_ltgr(obj, obj); // object shouldn't be null at this point
|
||||
branch_optimized(bcondAlways, slow);
|
||||
|
||||
bind(unlocked);
|
||||
}
|
||||
|
||||
void MacroAssembler::compiler_fast_lock_lightweight_object(Register obj, Register tmp1, Register tmp2) {
|
||||
assert_different_registers(obj, tmp1, tmp2);
|
||||
|
||||
// Handle inflated monitor.
|
||||
NearLabel inflated;
|
||||
// Finish fast lock successfully. MUST reach to with flag == NE
|
||||
NearLabel locked;
|
||||
// Finish fast lock unsuccessfully. MUST branch to with flag == EQ
|
||||
NearLabel slow_path;
|
||||
|
||||
if (DiagnoseSyncOnValueBasedClasses != 0) {
|
||||
load_klass(tmp1, obj);
|
||||
z_l(tmp1, Address(tmp1, Klass::access_flags_offset()));
|
||||
assert((JVM_ACC_IS_VALUE_BASED_CLASS & 0xFFFF) == 0, "or change following instruction");
|
||||
z_nilh(tmp1, JVM_ACC_IS_VALUE_BASED_CLASS >> 16);
|
||||
z_brne(slow_path);
|
||||
}
|
||||
|
||||
const Register mark = tmp1;
|
||||
const int mark_offset = oopDesc::mark_offset_in_bytes();
|
||||
const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset();
|
||||
|
||||
BLOCK_COMMENT("compiler_fast_lightweight_locking {");
|
||||
{ // lightweight locking
|
||||
|
||||
// Push lock to the lock stack and finish successfully. MUST reach to with flag == EQ
|
||||
NearLabel push;
|
||||
|
||||
const Register top = tmp2;
|
||||
|
||||
// Check if lock-stack is full.
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
compareU32_and_branch(top, (unsigned) LockStack::end_offset() - 1, bcondHigh, slow_path);
|
||||
|
||||
// The underflow check is elided. The recursive check will always fail
|
||||
// when the lock stack is empty because of the _bad_oop_sentinel field.
|
||||
|
||||
// Check if recursive.
|
||||
z_aghi(top, -oopSize);
|
||||
z_cg(obj, Address(Z_thread, top));
|
||||
z_bre(push);
|
||||
|
||||
// Check for monitor (0b10)
|
||||
z_lg(mark, Address(obj, mark_offset));
|
||||
z_tmll(mark, markWord::monitor_value);
|
||||
z_brnaz(inflated);
|
||||
|
||||
// not inflated
|
||||
|
||||
{ // Try to lock. Transition lock bits 0b01 => 0b00
|
||||
assert(mark_offset == 0, "required to avoid a lea");
|
||||
const Register locked_obj = top;
|
||||
z_oill(mark, markWord::unlocked_value);
|
||||
z_lgr(locked_obj, mark);
|
||||
// Clear lock-bits from locked_obj (locked state)
|
||||
z_xilf(locked_obj, markWord::unlocked_value);
|
||||
z_csg(mark, locked_obj, mark_offset, obj);
|
||||
branch_optimized(Assembler::bcondNotEqual, slow_path);
|
||||
}
|
||||
|
||||
bind(push);
|
||||
|
||||
// After successful lock, push object on lock-stack.
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
z_stg(obj, Address(Z_thread, top));
|
||||
z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize);
|
||||
|
||||
z_cgr(obj, obj); // set the CC to EQ, as it could be changed by alsi
|
||||
z_bru(locked);
|
||||
}
|
||||
BLOCK_COMMENT("} compiler_fast_lightweight_locking");
|
||||
|
||||
BLOCK_COMMENT("handle_inflated_monitor_lightweight_locking {");
|
||||
{ // Handle inflated monitor.
|
||||
bind(inflated);
|
||||
|
||||
// mark contains the tagged ObjectMonitor*.
|
||||
const Register tagged_monitor = mark;
|
||||
const Register zero = tmp2;
|
||||
|
||||
// Try to CAS m->owner from null to current thread.
|
||||
// If m->owner is null, then csg succeeds and sets m->owner=THREAD and CR=EQ.
|
||||
// Otherwise, register zero is filled with the current owner.
|
||||
z_lghi(zero, 0);
|
||||
z_csg(zero, Z_thread, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), tagged_monitor);
|
||||
z_bre(locked);
|
||||
|
||||
// Check if recursive.
|
||||
z_cgr(Z_thread, zero); // zero contains the owner from z_csg instruction
|
||||
z_brne(slow_path);
|
||||
|
||||
// Recursive
|
||||
z_agsi(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 1ll);
|
||||
z_cgr(zero, zero);
|
||||
// z_bru(locked);
|
||||
// Uncomment above line in the future, for now jump address is right next to us.
|
||||
}
|
||||
BLOCK_COMMENT("} handle_inflated_monitor_lightweight_locking");
|
||||
|
||||
bind(locked);
|
||||
|
||||
#ifdef ASSERT
|
||||
// Check that locked label is reached with flag == EQ.
|
||||
NearLabel flag_correct;
|
||||
z_bre(flag_correct);
|
||||
stop("CC is not set to EQ, it should be - lock");
|
||||
#endif // ASSERT
|
||||
|
||||
bind(slow_path);
|
||||
|
||||
#ifdef ASSERT
|
||||
// Check that slow_path label is reached with flag == NE.
|
||||
z_brne(flag_correct);
|
||||
stop("CC is not set to NE, it should be - lock");
|
||||
bind(flag_correct);
|
||||
#endif // ASSERT
|
||||
|
||||
// C2 uses the value of flag (NE vs EQ) to determine the continuation.
|
||||
}
|
||||
|
||||
void MacroAssembler::compiler_fast_unlock_lightweight_object(Register obj, Register tmp1, Register tmp2) {
|
||||
assert_different_registers(obj, tmp1, tmp2);
|
||||
|
||||
// Handle inflated monitor.
|
||||
NearLabel inflated, inflated_load_monitor;
|
||||
// Finish fast unlock successfully. MUST reach to with flag == EQ.
|
||||
NearLabel unlocked;
|
||||
// Finish fast unlock unsuccessfully. MUST branch to with flag == NE.
|
||||
NearLabel slow_path;
|
||||
|
||||
const Register mark = tmp1;
|
||||
const Register top = tmp2;
|
||||
const int mark_offset = oopDesc::mark_offset_in_bytes();
|
||||
const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset();
|
||||
|
||||
BLOCK_COMMENT("compiler_fast_lightweight_unlock {");
|
||||
{ // Lightweight Unlock
|
||||
|
||||
// Check if obj is top of lock-stack.
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
|
||||
z_aghi(top, -oopSize);
|
||||
z_cg(obj, Address(Z_thread, top));
|
||||
branch_optimized(bcondNotEqual, inflated_load_monitor);
|
||||
|
||||
// Pop lock-stack.
|
||||
#ifdef ASSERT
|
||||
const Register temp_top = tmp1; // let's not kill top here, we can use for recursive check
|
||||
z_agrk(temp_top, top, Z_thread);
|
||||
z_xc(0, oopSize-1, temp_top, 0, temp_top); // wipe out lock-stack entry
|
||||
#endif
|
||||
z_alsi(in_bytes(JavaThread::lock_stack_top_offset()), Z_thread, -oopSize); // pop object
|
||||
z_cr(tmp, tmp); // set CC to EQ
|
||||
z_alsi(in_bytes(ls_top_offset), Z_thread, -oopSize); // pop object
|
||||
|
||||
// The underflow check is elided. The recursive check will always fail
|
||||
// when the lock stack is empty because of the _bad_oop_sentinel field.
|
||||
|
||||
// Check if recursive.
|
||||
z_aghi(top, -oopSize);
|
||||
z_cg(obj, Address(Z_thread, top));
|
||||
z_bre(unlocked);
|
||||
|
||||
// Not recursive
|
||||
|
||||
// Check for monitor (0b10).
|
||||
z_lg(mark, Address(obj, mark_offset));
|
||||
z_tmll(mark, markWord::monitor_value);
|
||||
z_brnaz(inflated);
|
||||
|
||||
#ifdef ASSERT
|
||||
// Check header not unlocked (0b01).
|
||||
NearLabel not_unlocked;
|
||||
z_tmll(mark, markWord::unlocked_value);
|
||||
z_braz(not_unlocked);
|
||||
stop("lightweight_unlock already unlocked");
|
||||
bind(not_unlocked);
|
||||
#endif // ASSERT
|
||||
|
||||
{ // Try to unlock. Transition lock bits 0b00 => 0b01
|
||||
Register unlocked_obj = top;
|
||||
z_lgr(unlocked_obj, mark);
|
||||
z_oill(unlocked_obj, markWord::unlocked_value);
|
||||
z_csg(mark, unlocked_obj, mark_offset, obj);
|
||||
branch_optimized(Assembler::bcondEqual, unlocked);
|
||||
}
|
||||
|
||||
// Restore lock-stack and handle the unlock in runtime.
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
DEBUG_ONLY(z_stg(obj, Address(Z_thread, top));)
|
||||
z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize);
|
||||
// set CC to NE
|
||||
z_ltgr(obj, obj); // object is not null here
|
||||
z_bru(slow_path);
|
||||
}
|
||||
BLOCK_COMMENT("} compiler_fast_lightweight_unlock");
|
||||
|
||||
{ // Handle inflated monitor.
|
||||
|
||||
bind(inflated_load_monitor);
|
||||
|
||||
z_lg(mark, Address(obj, mark_offset));
|
||||
|
||||
#ifdef ASSERT
|
||||
z_tmll(mark, markWord::monitor_value);
|
||||
z_brnaz(inflated);
|
||||
stop("Fast Unlock not monitor");
|
||||
#endif // ASSERT
|
||||
|
||||
bind(inflated);
|
||||
|
||||
#ifdef ASSERT
|
||||
NearLabel check_done, loop;
|
||||
z_lgf(top, Address(Z_thread, ls_top_offset));
|
||||
bind(loop);
|
||||
z_aghi(top, -oopSize);
|
||||
compareU32_and_branch(top, in_bytes(JavaThread::lock_stack_base_offset()),
|
||||
bcondLow, check_done);
|
||||
z_cg(obj, Address(Z_thread, top));
|
||||
z_brne(loop);
|
||||
stop("Fast Unlock lock on stack");
|
||||
bind(check_done);
|
||||
#endif // ASSERT
|
||||
|
||||
// mark contains the tagged ObjectMonitor*.
|
||||
const Register monitor = mark;
|
||||
|
||||
NearLabel not_recursive;
|
||||
const Register recursions = tmp2;
|
||||
|
||||
// Check if recursive.
|
||||
load_and_test_long(recursions, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
|
||||
z_bre(not_recursive); // if 0 then jump, it's not recursive locking
|
||||
|
||||
// Recursive unlock
|
||||
z_agsi(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), -1ll);
|
||||
z_cgr(monitor, monitor); // set the CC to EQUAL
|
||||
z_bru(unlocked);
|
||||
|
||||
bind(not_recursive);
|
||||
|
||||
NearLabel not_ok;
|
||||
// Check if the entry lists are empty.
|
||||
load_and_test_long(tmp2, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
|
||||
z_brne(not_ok);
|
||||
load_and_test_long(tmp2, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
|
||||
z_brne(not_ok);
|
||||
|
||||
z_release();
|
||||
z_stg(tmp2 /*=0*/, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor);
|
||||
|
||||
z_bru(unlocked); // CC = EQ here
|
||||
|
||||
bind(not_ok);
|
||||
|
||||
// The owner may be anonymous, and we removed the last obj entry in
|
||||
// the lock-stack. This loses the information about the owner.
|
||||
// Write the thread to the owner field so the runtime knows the owner.
|
||||
z_stg(Z_thread, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor);
|
||||
z_bru(slow_path); // CC = NE here
|
||||
}
|
||||
|
||||
bind(unlocked);
|
||||
|
||||
#ifdef ASSERT
|
||||
// Check that unlocked label is reached with flag == EQ.
|
||||
NearLabel flag_correct;
|
||||
z_bre(flag_correct);
|
||||
stop("CC is not set to EQ, it should be - unlock");
|
||||
#endif // ASSERT
|
||||
|
||||
bind(slow_path);
|
||||
|
||||
#ifdef ASSERT
|
||||
// Check that slow_path label is reached with flag == NE.
|
||||
z_brne(flag_correct);
|
||||
stop("CC is not set to NE, it should be - unlock");
|
||||
bind(flag_correct);
|
||||
#endif // ASSERT
|
||||
|
||||
// C2 uses the value of flag (NE vs EQ) to determine the continuation.
|
||||
}
|
||||
|
||||
void MacroAssembler::pop_count_int(Register r_dst, Register r_src, Register r_tmp) {
|
||||
|
||||
@ -727,8 +727,10 @@ class MacroAssembler: public Assembler {
|
||||
|
||||
void compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2);
|
||||
void compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2);
|
||||
void lightweight_lock(Register obj, Register hdr, Register tmp, Label& slow);
|
||||
void lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow);
|
||||
void lightweight_lock(Register obj, Register tmp1, Register tmp2, Label& slow);
|
||||
void lightweight_unlock(Register obj, Register tmp1, Register tmp2, Label& slow);
|
||||
void compiler_fast_lock_lightweight_object(Register obj, Register tmp1, Register tmp2);
|
||||
void compiler_fast_unlock_lightweight_object(Register obj, Register tmp1, Register tmp2);
|
||||
|
||||
void resolve_jobject(Register value, Register tmp1, Register tmp2);
|
||||
|
||||
|
||||
@ -9579,6 +9579,7 @@ instruct partialSubtypeCheck_vs_zero(flagsReg pcc, rarg2RegP sub, rarg3RegP supe
|
||||
// inlined locking and unlocking
|
||||
|
||||
instruct cmpFastLock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{
|
||||
predicate(LockingMode != LM_LIGHTWEIGHT);
|
||||
match(Set pcc (FastLock oop box));
|
||||
effect(TEMP tmp1, TEMP tmp2);
|
||||
ins_cost(100);
|
||||
@ -9589,6 +9590,7 @@ instruct cmpFastLock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRe
|
||||
%}
|
||||
|
||||
instruct cmpFastUnlock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{
|
||||
predicate(LockingMode != LM_LIGHTWEIGHT);
|
||||
match(Set pcc (FastUnlock oop box));
|
||||
effect(TEMP tmp1, TEMP tmp2);
|
||||
ins_cost(100);
|
||||
@ -9598,6 +9600,38 @@ instruct cmpFastUnlock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, i
|
||||
ins_pipe(pipe_class_dummy);
|
||||
%}
|
||||
|
||||
instruct cmpFastLockLightweight(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{
|
||||
predicate(LockingMode == LM_LIGHTWEIGHT);
|
||||
match(Set pcc (FastLock oop box));
|
||||
effect(TEMP tmp1, TEMP tmp2);
|
||||
ins_cost(100);
|
||||
// TODO: s390 port size(VARIABLE_SIZE);
|
||||
format %{ "FASTLOCK $oop, $box; KILL Z_ARG4, Z_ARG5" %}
|
||||
ins_encode %{
|
||||
__ fast_lock_lightweight($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register);
|
||||
// If locking was successful, cc should indicate 'EQ'.
|
||||
// The compiler generates a branch to the runtime call to
|
||||
// _complete_monitor_locking_Java for the case where cc is 'NE'.
|
||||
%}
|
||||
ins_pipe(pipe_class_dummy);
|
||||
%}
|
||||
|
||||
instruct cmpFastUnlockLightweight(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{
|
||||
predicate(LockingMode == LM_LIGHTWEIGHT);
|
||||
match(Set pcc (FastUnlock oop box));
|
||||
effect(TEMP tmp1, TEMP tmp2);
|
||||
ins_cost(100);
|
||||
// TODO: s390 port size(FIXED_SIZE);
|
||||
format %{ "FASTUNLOCK $oop, $box; KILL Z_ARG4, Z_ARG5" %}
|
||||
ins_encode %{
|
||||
__ fast_unlock_lightweight($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register);
|
||||
// If unlocking was successful, cc should indicate 'EQ'.
|
||||
// The compiler generates a branch to the runtime call to
|
||||
// _complete_monitor_unlocking_Java for the case where cc is 'NE'.
|
||||
%}
|
||||
ins_pipe(pipe_class_dummy);
|
||||
%}
|
||||
|
||||
instruct inlineCallClearArrayConst(SSlenDW cnt, iRegP_N2P base, Universe dummy, flagsReg cr) %{
|
||||
match(Set dummy (ClearArray cnt base));
|
||||
effect(KILL cr);
|
||||
|
||||
@ -1711,8 +1711,13 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
|
||||
__ add2reg(r_box, lock_offset, Z_SP);
|
||||
|
||||
// Try fastpath for locking.
|
||||
// Fast_lock kills r_temp_1, r_temp_2.
|
||||
__ compiler_fast_lock_object(r_oop, r_box, r_tmp1, r_tmp2);
|
||||
if (LockingMode == LM_LIGHTWEIGHT) {
|
||||
// Fast_lock kills r_temp_1, r_temp_2.
|
||||
__ compiler_fast_lock_lightweight_object(r_oop, r_tmp1, r_tmp2);
|
||||
} else {
|
||||
// Fast_lock kills r_temp_1, r_temp_2.
|
||||
__ compiler_fast_lock_object(r_oop, r_box, r_tmp1, r_tmp2);
|
||||
}
|
||||
__ z_bre(done);
|
||||
|
||||
//-------------------------------------------------------------------------
|
||||
@ -1910,8 +1915,13 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
|
||||
__ add2reg(r_box, lock_offset, Z_SP);
|
||||
|
||||
// Try fastpath for unlocking.
|
||||
// Fast_unlock kills r_tmp1, r_tmp2.
|
||||
__ compiler_fast_unlock_object(r_oop, r_box, r_tmp1, r_tmp2);
|
||||
if (LockingMode == LM_LIGHTWEIGHT) {
|
||||
// Fast_unlock kills r_tmp1, r_tmp2.
|
||||
__ compiler_fast_unlock_lightweight_object(r_oop, r_tmp1, r_tmp2);
|
||||
} else {
|
||||
// Fast_unlock kills r_tmp1, r_tmp2.
|
||||
__ compiler_fast_unlock_object(r_oop, r_box, r_tmp1, r_tmp2);
|
||||
}
|
||||
__ z_bre(done);
|
||||
|
||||
// Slow path for unlocking.
|
||||
|
||||
@ -413,6 +413,8 @@ class VM_Version: public Abstract_VM_Version {
|
||||
// s390 supports fast class initialization checks
|
||||
static bool supports_fast_class_init_checks() { return true; }
|
||||
|
||||
constexpr static bool supports_recursive_lightweight_locking() { return true; }
|
||||
|
||||
// CPU feature query functions
|
||||
static const char* get_model_string() { return _model_string; }
|
||||
static bool has_StoreFacilityListExtended() { return (_features[0] & StoreFacilityListExtendedMask) == StoreFacilityListExtendedMask; }
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user