From b48dbf6bfa652ef031c35f0a85a409142563aa72 Mon Sep 17 00:00:00 2001 From: Stefan Karlsson Date: Thu, 14 Sep 2023 11:50:24 +0000 Subject: [PATCH] 8316181: Move the fast locking implementation out of the .ad files Reviewed-by: thartmann, rkennke, fyang --- src/hotspot/cpu/aarch64/aarch64.ad | 204 +--------------- .../cpu/aarch64/c2_MacroAssembler_aarch64.cpp | 196 +++++++++++++++ .../cpu/aarch64/c2_MacroAssembler_aarch64.hpp | 5 + .../cpu/riscv/c2_MacroAssembler_riscv.cpp | 217 +++++++++++++++++ .../cpu/riscv/c2_MacroAssembler_riscv.hpp | 5 + src/hotspot/cpu/riscv/riscv.ad | 225 +----------------- 6 files changed, 435 insertions(+), 417 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 2f83b2d99c7..6f56d0e8386 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -3816,202 +3816,6 @@ encode %{ __ br(target_reg); %} - enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{ - C2_MacroAssembler _masm(&cbuf); - Register oop = as_Register($object$$reg); - Register box = as_Register($box$$reg); - Register disp_hdr = as_Register($tmp$$reg); - Register tmp = as_Register($tmp2$$reg); - Label cont; - Label object_has_monitor; - Label count, no_count; - - assert_different_registers(oop, box, tmp, disp_hdr); - - // Load markWord from object into displaced_header. - __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes())); - - if (DiagnoseSyncOnValueBasedClasses != 0) { - __ load_klass(tmp, oop); - __ ldrw(tmp, Address(tmp, Klass::access_flags_offset())); - __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS); - __ br(Assembler::NE, cont); - } - - // Check for existing monitor - __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor); - - if (LockingMode == LM_MONITOR) { - __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0. - __ b(cont); - } else if (LockingMode == LM_LEGACY) { - // Set tmp to be (markWord of object | UNLOCK_VALUE). - __ orr(tmp, disp_hdr, markWord::unlocked_value); - - // Initialize the box. (Must happen before we update the object mark!) - __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); - - // Compare object markWord with an unlocked value (tmp) and if - // equal exchange the stack address of our box with object markWord. - // On failure disp_hdr contains the possibly locked markWord. - __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true, - /*release*/ true, /*weak*/ false, disp_hdr); - __ br(Assembler::EQ, cont); - - assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); - - // If the compare-and-exchange succeeded, then we found an unlocked - // object, will have now locked it will continue at label cont - - // Check if the owner is self by comparing the value in the - // markWord of object (disp_hdr) with the stack pointer. - __ mov(rscratch1, sp); - __ sub(disp_hdr, disp_hdr, rscratch1); - __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place)); - // If condition is true we are cont and hence we can store 0 as the - // displaced header in the box, which indicates that it is a recursive lock. - __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result - __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes())); - __ b(cont); - } else { - assert(LockingMode == LM_LIGHTWEIGHT, "must be"); - __ lightweight_lock(oop, disp_hdr, tmp, rscratch1, no_count); - __ b(count); - } - - // Handle existing monitor. - __ bind(object_has_monitor); - - // The object's monitor m is unlocked iff m->owner == NULL, - // otherwise m->owner may contain a thread or a stack address. - // - // Try to CAS m->owner from NULL to current thread. - __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value)); - __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true, - /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result - - if (LockingMode != LM_LIGHTWEIGHT) { - // Store a non-null value into the box to avoid looking like a re-entrant - // lock. The fast-path monitor unlock code checks for - // markWord::monitor_value so use markWord::unused_mark which has the - // relevant bit set, and also matches ObjectSynchronizer::enter. - __ mov(tmp, (address)markWord::unused_mark().value()); - __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); - } - __ br(Assembler::EQ, cont); // CAS success means locking succeeded - - __ cmp(rscratch1, rthread); - __ br(Assembler::NE, cont); // Check for recursive locking - - // Recursive lock case - __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1); - // flag == EQ still from the cmp above, checking if this is a reentrant lock - - __ bind(cont); - // flag == EQ indicates success - // flag == NE indicates failure - __ br(Assembler::NE, no_count); - - __ bind(count); - __ increment(Address(rthread, JavaThread::held_monitor_count_offset())); - - __ bind(no_count); - %} - - enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{ - C2_MacroAssembler _masm(&cbuf); - Register oop = as_Register($object$$reg); - Register box = as_Register($box$$reg); - Register disp_hdr = as_Register($tmp$$reg); - Register tmp = as_Register($tmp2$$reg); - Label cont; - Label object_has_monitor; - Label count, no_count; - - assert_different_registers(oop, box, tmp, disp_hdr); - - if (LockingMode == LM_LEGACY) { - // Find the lock address and load the displaced header from the stack. - __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes())); - - // If the displaced header is 0, we have a recursive unlock. - __ cmp(disp_hdr, zr); - __ br(Assembler::EQ, cont); - } - - // Handle existing monitor. - __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes())); - __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor); - - if (LockingMode == LM_MONITOR) { - __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0. - __ b(cont); - } else if (LockingMode == LM_LEGACY) { - // Check if it is still a light weight lock, this is is true if we - // see the stack address of the basicLock in the markWord of the - // object. - - __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false, - /*release*/ true, /*weak*/ false, tmp); - __ b(cont); - } else { - assert(LockingMode == LM_LIGHTWEIGHT, "must be"); - __ lightweight_unlock(oop, tmp, box, disp_hdr, no_count); - __ b(count); - } - - assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); - - // Handle existing monitor. - __ bind(object_has_monitor); - STATIC_ASSERT(markWord::monitor_value <= INT_MAX); - __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor - - if (LockingMode == LM_LIGHTWEIGHT) { - // If the owner is anonymous, we need to fix it -- in an outline stub. - Register tmp2 = disp_hdr; - __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset())); - // We cannot use tbnz here, the target might be too far away and cannot - // be encoded. - __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER); - C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2); - Compile::current()->output()->add_stub(stub); - __ br(Assembler::NE, stub->entry()); - __ bind(stub->continuation()); - } - - __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset())); - - Label notRecursive; - __ cbz(disp_hdr, notRecursive); - - // Recursive lock - __ sub(disp_hdr, disp_hdr, 1u); - __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset())); - __ cmp(disp_hdr, disp_hdr); // Sets flags for result - __ b(cont); - - __ bind(notRecursive); - __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset())); - __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset())); - __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0. - __ cmp(rscratch1, zr); // Sets flags for result - __ cbnz(rscratch1, cont); - // need a release store here - __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset())); - __ stlr(zr, tmp); // set unowned - - __ bind(cont); - // flag == EQ indicates success - // flag == NE indicates failure - __ br(Assembler::NE, no_count); - - __ bind(count); - __ decrement(Address(rthread, JavaThread::held_monitor_count_offset())); - - __ bind(no_count); - %} - %} //----------FRAME-------------------------------------------------------------- @@ -16626,7 +16430,9 @@ instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegP ins_cost(5 * INSN_COST); format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %} - ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2)); + ins_encode %{ + __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register); + %} ins_pipe(pipe_serial); %} @@ -16639,7 +16445,9 @@ instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRe ins_cost(5 * INSN_COST); format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %} - ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2)); + ins_encode %{ + __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register); + %} ins_pipe(pipe_serial); %} diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp index dbe64f8f9ca..8890b5a3411 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp @@ -45,6 +45,202 @@ typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr); +void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmpReg, + Register tmp2Reg) { + Register oop = objectReg; + Register box = boxReg; + Register disp_hdr = tmpReg; + Register tmp = tmp2Reg; + Label cont; + Label object_has_monitor; + Label count, no_count; + + assert_different_registers(oop, box, tmp, disp_hdr); + + // Load markWord from object into displaced_header. + ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes())); + + if (DiagnoseSyncOnValueBasedClasses != 0) { + load_klass(tmp, oop); + ldrw(tmp, Address(tmp, Klass::access_flags_offset())); + tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS); + br(Assembler::NE, cont); + } + + // Check for existing monitor + tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor); + + if (LockingMode == LM_MONITOR) { + tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0. + b(cont); + } else if (LockingMode == LM_LEGACY) { + // Set tmp to be (markWord of object | UNLOCK_VALUE). + orr(tmp, disp_hdr, markWord::unlocked_value); + + // Initialize the box. (Must happen before we update the object mark!) + str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); + + // Compare object markWord with an unlocked value (tmp) and if + // equal exchange the stack address of our box with object markWord. + // On failure disp_hdr contains the possibly locked markWord. + cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true, + /*release*/ true, /*weak*/ false, disp_hdr); + br(Assembler::EQ, cont); + + assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); + + // If the compare-and-exchange succeeded, then we found an unlocked + // object, will have now locked it will continue at label cont + + // Check if the owner is self by comparing the value in the + // markWord of object (disp_hdr) with the stack pointer. + mov(rscratch1, sp); + sub(disp_hdr, disp_hdr, rscratch1); + mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place)); + // If condition is true we are cont and hence we can store 0 as the + // displaced header in the box, which indicates that it is a recursive lock. + ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result + str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes())); + b(cont); + } else { + assert(LockingMode == LM_LIGHTWEIGHT, "must be"); + lightweight_lock(oop, disp_hdr, tmp, rscratch1, no_count); + b(count); + } + + // Handle existing monitor. + bind(object_has_monitor); + + // The object's monitor m is unlocked iff m->owner == NULL, + // otherwise m->owner may contain a thread or a stack address. + // + // Try to CAS m->owner from NULL to current thread. + add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset())-markWord::monitor_value)); + cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true, + /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result + + if (LockingMode != LM_LIGHTWEIGHT) { + // Store a non-null value into the box to avoid looking like a re-entrant + // lock. The fast-path monitor unlock code checks for + // markWord::monitor_value so use markWord::unused_mark which has the + // relevant bit set, and also matches ObjectSynchronizer::enter. + mov(tmp, (address)markWord::unused_mark().value()); + str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); + } + br(Assembler::EQ, cont); // CAS success means locking succeeded + + cmp(rscratch1, rthread); + br(Assembler::NE, cont); // Check for recursive locking + + // Recursive lock case + increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1); + // flag == EQ still from the cmp above, checking if this is a reentrant lock + + bind(cont); + // flag == EQ indicates success + // flag == NE indicates failure + br(Assembler::NE, no_count); + + bind(count); + increment(Address(rthread, JavaThread::held_monitor_count_offset())); + + bind(no_count); +} + +void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Register tmpReg, + Register tmp2Reg) { + Register oop = objectReg; + Register box = boxReg; + Register disp_hdr = tmpReg; + Register tmp = tmp2Reg; + Label cont; + Label object_has_monitor; + Label count, no_count; + + assert_different_registers(oop, box, tmp, disp_hdr); + + if (LockingMode == LM_LEGACY) { + // Find the lock address and load the displaced header from the stack. + ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes())); + + // If the displaced header is 0, we have a recursive unlock. + cmp(disp_hdr, zr); + br(Assembler::EQ, cont); + } + + // Handle existing monitor. + ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes())); + tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor); + + if (LockingMode == LM_MONITOR) { + tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0. + b(cont); + } else if (LockingMode == LM_LEGACY) { + // Check if it is still a light weight lock, this is is true if we + // see the stack address of the basicLock in the markWord of the + // object. + + cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false, + /*release*/ true, /*weak*/ false, tmp); + b(cont); + } else { + assert(LockingMode == LM_LIGHTWEIGHT, "must be"); + lightweight_unlock(oop, tmp, box, disp_hdr, no_count); + b(count); + } + + assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); + + // Handle existing monitor. + bind(object_has_monitor); + STATIC_ASSERT(markWord::monitor_value <= INT_MAX); + add(tmp, tmp, -(int)markWord::monitor_value); // monitor + + if (LockingMode == LM_LIGHTWEIGHT) { + // If the owner is anonymous, we need to fix it -- in an outline stub. + Register tmp2 = disp_hdr; + ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset())); + // We cannot use tbnz here, the target might be too far away and cannot + // be encoded. + tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER); + C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2); + Compile::current()->output()->add_stub(stub); + br(Assembler::NE, stub->entry()); + bind(stub->continuation()); + } + + ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset())); + + Label notRecursive; + cbz(disp_hdr, notRecursive); + + // Recursive lock + sub(disp_hdr, disp_hdr, 1u); + str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset())); + cmp(disp_hdr, disp_hdr); // Sets flags for result + b(cont); + + bind(notRecursive); + ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset())); + ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset())); + orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0. + cmp(rscratch1, zr); // Sets flags for result + cbnz(rscratch1, cont); + // need a release store here + lea(tmp, Address(tmp, ObjectMonitor::owner_offset())); + stlr(zr, tmp); // set unowned + + bind(cont); + // flag == EQ indicates success + // flag == NE indicates failure + br(Assembler::NE, no_count); + + bind(count); + decrement(Address(rthread, JavaThread::held_monitor_count_offset())); + + bind(no_count); +} + // Search for str1 in str2 and return index or -1 // Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1. void C2_MacroAssembler::string_indexof(Register str2, Register str1, diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp index ccfd60b1a8b..42324692de7 100644 --- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp @@ -35,6 +35,11 @@ enum shift_kind kind = Assembler::LSL, unsigned shift = 0); public: + // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file. + // See full description in macroAssembler_aarch64.cpp. + void fast_lock(Register object, Register box, Register tmp, Register tmp2); + void fast_unlock(Register object, Register box, Register tmp, Register tmp2); + void string_compare(Register str1, Register str2, Register cnt1, Register cnt2, Register result, Register tmp1, Register tmp2, FloatRegister vtmp1, diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp index dfb333b7c78..c435ee66f84 100644 --- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp @@ -43,6 +43,223 @@ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") +void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmp1Reg, + Register tmp2Reg) { + // Use cr register to indicate the fast_lock result: zero for success; non-zero for failure. + Register flag = t1; + Register oop = objectReg; + Register box = boxReg; + Register disp_hdr = tmp1Reg; + Register tmp = tmp2Reg; + Label cont; + Label object_has_monitor; + Label count, no_count; + + assert_different_registers(oop, box, tmp, disp_hdr, t0); + + // Load markWord from object into displaced_header. + ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes())); + + if (DiagnoseSyncOnValueBasedClasses != 0) { + load_klass(flag, oop); + lwu(flag, Address(flag, Klass::access_flags_offset())); + test_bit(flag, flag, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS), tmp /* tmp */); + bnez(flag, cont, true /* is_far */); + } + + // Check for existing monitor + test_bit(t0, disp_hdr, exact_log2(markWord::monitor_value)); + bnez(t0, object_has_monitor); + + if (LockingMode == LM_MONITOR) { + mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path + j(cont); + } else if (LockingMode == LM_LEGACY) { + // Set tmp to be (markWord of object | UNLOCK_VALUE). + ori(tmp, disp_hdr, markWord::unlocked_value); + + // Initialize the box. (Must happen before we update the object mark!) + sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); + + // Compare object markWord with an unlocked value (tmp) and if + // equal exchange the stack address of our box with object markWord. + // On failure disp_hdr contains the possibly locked markWord. + cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64, Assembler::aq, + Assembler::rl, /*result*/disp_hdr); + mv(flag, zr); + beq(disp_hdr, tmp, cont); // prepare zero flag and goto cont if we won the cas + + assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); + + // If the compare-and-exchange succeeded, then we found an unlocked + // object, will have now locked it will continue at label cont + // We did not see an unlocked object so try the fast recursive case. + + // Check if the owner is self by comparing the value in the + // markWord of object (disp_hdr) with the stack pointer. + sub(disp_hdr, disp_hdr, sp); + mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place)); + // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto cont, + // hence we can store 0 as the displaced header in the box, which indicates that it is a + // recursive lock. + andr(tmp/*==0?*/, disp_hdr, tmp); + sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes())); + mv(flag, tmp); // we can use the value of tmp as the result here + j(cont); + } else { + assert(LockingMode == LM_LIGHTWEIGHT, ""); + Label slow; + lightweight_lock(oop, disp_hdr, tmp, t0, slow); + + // Indicate success on completion. + mv(flag, zr); + j(count); + bind(slow); + mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path + j(no_count); + } + + // Handle existing monitor. + bind(object_has_monitor); + // The object's monitor m is unlocked iff m->owner == NULL, + // otherwise m->owner may contain a thread or a stack address. + // + // Try to CAS m->owner from NULL to current thread. + add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value)); + cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, Assembler::aq, + Assembler::rl, /*result*/flag); // cas succeeds if flag == zr(expected) + + if (LockingMode != LM_LIGHTWEIGHT) { + // Store a non-null value into the box to avoid looking like a re-entrant + // lock. The fast-path monitor unlock code checks for + // markWord::monitor_value so use markWord::unused_mark which has the + // relevant bit set, and also matches ObjectSynchronizer::slow_enter. + mv(tmp, (address)markWord::unused_mark().value()); + sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); + } + + beqz(flag, cont); // CAS success means locking succeeded + + bne(flag, xthread, cont); // Check for recursive locking + + // Recursive lock case + mv(flag, zr); + increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1, t0, tmp); + + bind(cont); + // zero flag indicates success + // non-zero flag indicates failure + bnez(flag, no_count); + + bind(count); + increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp); + + bind(no_count); +} + +void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Register tmp1Reg, + Register tmp2Reg) { + // Use cr register to indicate the fast_unlock result: zero for success; non-zero for failure. + Register flag = t1; + Register oop = objectReg; + Register box = boxReg; + Register disp_hdr = tmp1Reg; + Register tmp = tmp2Reg; + Label cont; + Label object_has_monitor; + Label count, no_count; + + assert_different_registers(oop, box, tmp, disp_hdr, flag); + + if (LockingMode == LM_LEGACY) { + // Find the lock address and load the displaced header from the stack. + ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes())); + + // If the displaced header is 0, we have a recursive unlock. + mv(flag, disp_hdr); + beqz(disp_hdr, cont); + } + + // Handle existing monitor. + ld(tmp, Address(oop, oopDesc::mark_offset_in_bytes())); + test_bit(t0, tmp, exact_log2(markWord::monitor_value)); + bnez(t0, object_has_monitor); + + if (LockingMode == LM_MONITOR) { + mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path + j(cont); + } else if (LockingMode == LM_LEGACY) { + // Check if it is still a light weight lock, this is true if we + // see the stack address of the basicLock in the markWord of the + // object. + + cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64, Assembler::relaxed, + Assembler::rl, /*result*/tmp); + xorr(flag, box, tmp); // box == tmp if cas succeeds + j(cont); + } else { + assert(LockingMode == LM_LIGHTWEIGHT, ""); + Label slow; + lightweight_unlock(oop, tmp, box, disp_hdr, slow); + + // Indicate success on completion. + mv(flag, zr); + j(count); + bind(slow); + mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path + j(no_count); + } + + assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); + + // Handle existing monitor. + bind(object_has_monitor); + STATIC_ASSERT(markWord::monitor_value <= INT_MAX); + add(tmp, tmp, -(int)markWord::monitor_value); // monitor + + if (LockingMode == LM_LIGHTWEIGHT) { + // If the owner is anonymous, we need to fix it -- in an outline stub. + Register tmp2 = disp_hdr; + ld(tmp2, Address(tmp, ObjectMonitor::owner_offset())); + test_bit(t0, tmp2, exact_log2(ObjectMonitor::ANONYMOUS_OWNER)); + C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2); + Compile::current()->output()->add_stub(stub); + bnez(t0, stub->entry(), /* is_far */ true); + bind(stub->continuation()); + } + + ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset())); + + Label notRecursive; + beqz(disp_hdr, notRecursive); // Will be 0 if not recursive. + + // Recursive lock + addi(disp_hdr, disp_hdr, -1); + sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset())); + mv(flag, zr); + j(cont); + + bind(notRecursive); + ld(flag, Address(tmp, ObjectMonitor::EntryList_offset())); + ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset())); + orr(flag, flag, disp_hdr); // Will be 0 if both are 0. + bnez(flag, cont); + // need a release store here + la(tmp, Address(tmp, ObjectMonitor::owner_offset())); + membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); + sd(zr, Address(tmp)); // set unowned + + bind(cont); + // zero flag indicates success + // non-zero flag indicates failure + bnez(flag, no_count); + + bind(count); + decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp); + + bind(no_count); +} + // short string // StringUTF16.indexOfChar // StringLatin1.indexOfChar diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp index f8f3968c587..fdbdb4a2ac5 100644 --- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp @@ -39,6 +39,11 @@ VectorRegister vrs, bool is_latin, Label& DONE); public: + // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file. + // See full description in macroAssembler_riscv.cpp. + void fast_lock(Register object, Register box, Register tmp1, Register tmp2); + void fast_unlock(Register object, Register box, Register tmp1, Register tmp2); + void string_compare(Register str1, Register str2, Register cnt1, Register cnt2, Register result, Register tmp1, Register tmp2, Register tmp3, diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 98d016c3668..caf39498cf9 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -2436,223 +2436,6 @@ encode %{ } %} - // Use cr register to indicate the fast_lock result: zero for success; non-zero for failure. - enc_class riscv_enc_fast_lock(iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) %{ - C2_MacroAssembler _masm(&cbuf); - Register flag = t1; - Register oop = as_Register($object$$reg); - Register box = as_Register($box$$reg); - Register disp_hdr = as_Register($tmp1$$reg); - Register tmp = as_Register($tmp2$$reg); - Label cont; - Label object_has_monitor; - Label count, no_count; - - assert_different_registers(oop, box, tmp, disp_hdr, t0); - - // Load markWord from object into displaced_header. - __ ld(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes())); - - if (DiagnoseSyncOnValueBasedClasses != 0) { - __ load_klass(flag, oop); - __ lwu(flag, Address(flag, Klass::access_flags_offset())); - __ test_bit(flag, flag, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS), tmp /* tmp */); - __ bnez(flag, cont, true /* is_far */); - } - - // Check for existing monitor - __ test_bit(t0, disp_hdr, exact_log2(markWord::monitor_value)); - __ bnez(t0, object_has_monitor); - - if (LockingMode == LM_MONITOR) { - __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path - __ j(cont); - } else if (LockingMode == LM_LEGACY) { - // Set tmp to be (markWord of object | UNLOCK_VALUE). - __ ori(tmp, disp_hdr, markWord::unlocked_value); - - // Initialize the box. (Must happen before we update the object mark!) - __ sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); - - // Compare object markWord with an unlocked value (tmp) and if - // equal exchange the stack address of our box with object markWord. - // On failure disp_hdr contains the possibly locked markWord. - __ cmpxchg(/*memory address*/oop, /*expected value*/tmp, /*new value*/box, Assembler::int64, Assembler::aq, - Assembler::rl, /*result*/disp_hdr); - __ mv(flag, zr); - __ beq(disp_hdr, tmp, cont); // prepare zero flag and goto cont if we won the cas - - assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); - - // If the compare-and-exchange succeeded, then we found an unlocked - // object, will have now locked it will continue at label cont - // We did not see an unlocked object so try the fast recursive case. - - // Check if the owner is self by comparing the value in the - // markWord of object (disp_hdr) with the stack pointer. - __ sub(disp_hdr, disp_hdr, sp); - __ mv(tmp, (intptr_t) (~(os::vm_page_size()-1) | (uintptr_t)markWord::lock_mask_in_place)); - // If (mark & lock_mask) == 0 and mark - sp < page_size, we are stack-locking and goto cont, - // hence we can store 0 as the displaced header in the box, which indicates that it is a - // recursive lock. - __ andr(tmp/*==0?*/, disp_hdr, tmp); - __ sd(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes())); - __ mv(flag, tmp); // we can use the value of tmp as the result here - __ j(cont); - } else { - assert(LockingMode == LM_LIGHTWEIGHT, ""); - Label slow; - __ lightweight_lock(oop, disp_hdr, tmp, t0, slow); - - // Indicate success on completion. - __ mv(flag, zr); - __ j(count); - __ bind(slow); - __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow-path - __ j(no_count); - } - - // Handle existing monitor. - __ bind(object_has_monitor); - // The object's monitor m is unlocked iff m->owner == NULL, - // otherwise m->owner may contain a thread or a stack address. - // - // Try to CAS m->owner from NULL to current thread. - __ add(tmp, disp_hdr, (in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value)); - __ cmpxchg(/*memory address*/tmp, /*expected value*/zr, /*new value*/xthread, Assembler::int64, Assembler::aq, - Assembler::rl, /*result*/flag); // cas succeeds if flag == zr(expected) - - if (LockingMode != LM_LIGHTWEIGHT) { - // Store a non-null value into the box to avoid looking like a re-entrant - // lock. The fast-path monitor unlock code checks for - // markWord::monitor_value so use markWord::unused_mark which has the - // relevant bit set, and also matches ObjectSynchronizer::slow_enter. - __ mv(tmp, (address)markWord::unused_mark().value()); - __ sd(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes())); - } - - __ beqz(flag, cont); // CAS success means locking succeeded - - __ bne(flag, xthread, cont); // Check for recursive locking - - // Recursive lock case - __ mv(flag, zr); - __ increment(Address(disp_hdr, in_bytes(ObjectMonitor::recursions_offset()) - markWord::monitor_value), 1, t0, tmp); - - __ bind(cont); - // zero flag indicates success - // non-zero flag indicates failure - __ bnez(flag, no_count); - - __ bind(count); - __ increment(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp); - - __ bind(no_count); - %} - - // Use cr register to indicate the fast_unlock result: zero for success; non-zero for failure. - enc_class riscv_enc_fast_unlock(iRegP object, iRegP box, iRegPNoSp tmp1, iRegPNoSp tmp2) %{ - C2_MacroAssembler _masm(&cbuf); - Register flag = t1; - Register oop = as_Register($object$$reg); - Register box = as_Register($box$$reg); - Register disp_hdr = as_Register($tmp1$$reg); - Register tmp = as_Register($tmp2$$reg); - Label cont; - Label object_has_monitor; - Label count, no_count; - - assert_different_registers(oop, box, tmp, disp_hdr, flag); - - if (LockingMode == LM_LEGACY) { - // Find the lock address and load the displaced header from the stack. - __ ld(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes())); - - // If the displaced header is 0, we have a recursive unlock. - __ mv(flag, disp_hdr); - __ beqz(disp_hdr, cont); - } - - // Handle existing monitor. - __ ld(tmp, Address(oop, oopDesc::mark_offset_in_bytes())); - __ test_bit(t0, tmp, exact_log2(markWord::monitor_value)); - __ bnez(t0, object_has_monitor); - - if (LockingMode == LM_MONITOR) { - __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path - __ j(cont); - } else if (LockingMode == LM_LEGACY) { - // Check if it is still a light weight lock, this is true if we - // see the stack address of the basicLock in the markWord of the - // object. - - __ cmpxchg(/*memory address*/oop, /*expected value*/box, /*new value*/disp_hdr, Assembler::int64, Assembler::relaxed, - Assembler::rl, /*result*/tmp); - __ xorr(flag, box, tmp); // box == tmp if cas succeeds - __ j(cont); - } else { - assert(LockingMode == LM_LIGHTWEIGHT, ""); - Label slow; - __ lightweight_unlock(oop, tmp, box, disp_hdr, slow); - - // Indicate success on completion. - __ mv(flag, zr); - __ j(count); - __ bind(slow); - __ mv(flag, 1); // Set non-zero flag to indicate 'failure' -> take slow path - __ j(no_count); - } - - assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); - - // Handle existing monitor. - __ bind(object_has_monitor); - STATIC_ASSERT(markWord::monitor_value <= INT_MAX); - __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor - - if (LockingMode == LM_LIGHTWEIGHT) { - // If the owner is anonymous, we need to fix it -- in an outline stub. - Register tmp2 = disp_hdr; - __ ld(tmp2, Address(tmp, ObjectMonitor::owner_offset())); - __ test_bit(t0, tmp2, exact_log2(ObjectMonitor::ANONYMOUS_OWNER)); - C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2); - Compile::current()->output()->add_stub(stub); - __ bnez(t0, stub->entry(), /* is_far */ true); - __ bind(stub->continuation()); - } - - __ ld(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset())); - - Label notRecursive; - __ beqz(disp_hdr, notRecursive); // Will be 0 if not recursive. - - // Recursive lock - __ addi(disp_hdr, disp_hdr, -1); - __ sd(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset())); - __ mv(flag, zr); - __ j(cont); - - __ bind(notRecursive); - __ ld(flag, Address(tmp, ObjectMonitor::EntryList_offset())); - __ ld(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset())); - __ orr(flag, flag, disp_hdr); // Will be 0 if both are 0. - __ bnez(flag, cont); - // need a release store here - __ la(tmp, Address(tmp, ObjectMonitor::owner_offset())); - __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); - __ sd(zr, Address(tmp)); // set unowned - - __ bind(cont); - // zero flag indicates success - // non-zero flag indicates failure - __ bnez(flag, no_count); - - __ bind(count); - __ decrement(Address(xthread, JavaThread::held_monitor_count_offset()), 1, t0, tmp); - - __ bind(no_count); - %} - // arithmetic encodings enc_class riscv_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{ @@ -10416,7 +10199,9 @@ instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iReg ins_cost(LOAD_COST * 2 + STORE_COST * 3 + ALU_COST * 6 + BRANCH_COST * 3); format %{ "fastlock $object,$box\t! kills $tmp1,$tmp2, #@cmpFastLock" %} - ins_encode(riscv_enc_fast_lock(object, box, tmp1, tmp2)); + ins_encode %{ + __ fast_lock($object$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register); + %} ins_pipe(pipe_serial); %} @@ -10430,7 +10215,9 @@ instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp1, iR ins_cost(LOAD_COST * 2 + STORE_COST + ALU_COST * 2 + BRANCH_COST * 4); format %{ "fastunlock $object,$box\t! kills $tmp1, $tmp2, #@cmpFastUnlock" %} - ins_encode(riscv_enc_fast_unlock(object, box, tmp1, tmp2)); + ins_encode %{ + __ fast_unlock($object$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register); + %} ins_pipe(pipe_serial); %}