diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index 81d3338638d..cf3c851a7ec 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -3313,14 +3313,11 @@ void MacroAssembler::store_conditional(Register dst, } -void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, - Register new_val, +void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val, enum operand_size size, - Register tmp1, Register tmp2, Register tmp3) { + Register shift, Register mask, Register aligned_addr) { assert(size == int8 || size == int16, "unsupported operand size"); - Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3; - andi(shift, addr, 3); slli(shift, shift, 3); @@ -3335,8 +3332,6 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte } sll(mask, mask, shift); - notr(not_mask, mask); - sll(expected, expected, shift); andr(expected, expected, mask); @@ -3353,35 +3348,46 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result, bool result_as_bool, Register tmp1, Register tmp2, Register tmp3) { - Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0; - assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp); - cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3); + assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1); + + Register scratch0 = t0, aligned_addr = t1; + Register shift = tmp1, mask = tmp2, scratch1 = tmp3; + + cmpxchg_narrow_value_helper(addr, expected, new_val, size, shift, mask, aligned_addr); Label retry, fail, done; - bind(retry); - if (UseZacas) { - lw(old, aligned_addr); + lw(result, aligned_addr); - // if old & mask != expected - andr(tmp, old, mask); - bne(tmp, expected, fail); + bind(retry); // amocas loads the current value into result + notr(scratch1, mask); - andr(tmp, old, not_mask); - orr(tmp, tmp, new_val); + andr(scratch0, result, scratch1); // scratch0 = word - cas bits + orr(scratch1, expected, scratch0); // scratch1 = non-cas bits + cas bits + bne(result, scratch1, fail); // cas bits differ, cas failed - atomic_cas(old, tmp, aligned_addr, operand_size::int32, acquire, release); - bne(tmp, old, retry); + // result is the same as expected, use as expected value. + + // scratch0 is still = word - cas bits + // Or in the new value to create complete new value. + orr(scratch0, scratch0, new_val); + + mv(scratch1, result); // save our expected value + atomic_cas(result, scratch0, aligned_addr, operand_size::int32, acquire, release); + bne(scratch1, result, retry); } else { - lr_w(old, aligned_addr, acquire); - andr(tmp, old, mask); - bne(tmp, expected, fail); + notr(scratch1, mask); + bind(retry); - andr(tmp, old, not_mask); - orr(tmp, tmp, new_val); - sc_w(tmp, tmp, aligned_addr, release); - bnez(tmp, retry); + lr_w(result, aligned_addr, acquire); + andr(scratch0, result, mask); + bne(scratch0, expected, fail); + + andr(scratch0, result, scratch1); // scratch1 is ~mask + orr(scratch0, scratch0, new_val); + sc_w(scratch0, scratch0, aligned_addr, release); + bnez(scratch0, retry); } if (result_as_bool) { @@ -3393,10 +3399,10 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected, bind(done); } else { - andr(tmp, old, mask); - bind(fail); - srl(result, tmp, shift); + + andr(scratch0, result, mask); + srl(result, scratch0, shift); if (size == int8) { sign_extend(result, result, 8); @@ -3416,33 +3422,44 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result, Register tmp1, Register tmp2, Register tmp3) { - Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0; - assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp); - cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3); + assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1); + + Register scratch0 = t0, aligned_addr = t1; + Register shift = tmp1, mask = tmp2, scratch1 = tmp3; + + cmpxchg_narrow_value_helper(addr, expected, new_val, size, shift, mask, aligned_addr); Label fail, done; if (UseZacas) { - lw(old, aligned_addr); + lw(result, aligned_addr); - // if old & mask != expected - andr(tmp, old, mask); - bne(tmp, expected, fail); + notr(scratch1, mask); - andr(tmp, old, not_mask); - orr(tmp, tmp, new_val); + andr(scratch0, result, scratch1); // scratch0 = word - cas bits + orr(scratch1, expected, scratch0); // scratch1 = non-cas bits + cas bits + bne(result, scratch1, fail); // cas bits differ, cas failed - atomic_cas(tmp, new_val, addr, operand_size::int32, acquire, release); - bne(tmp, old, fail); + // result is the same as expected, use as expected value. + + // scratch0 is still = word - cas bits + // Or in the new value to create complete new value. + orr(scratch0, scratch0, new_val); + + mv(scratch1, result); // save our expected value + atomic_cas(result, scratch0, aligned_addr, operand_size::int32, acquire, release); + bne(scratch1, result, fail); // This weak, so just bail-out. } else { - lr_w(old, aligned_addr, acquire); - andr(tmp, old, mask); - bne(tmp, expected, fail); + notr(scratch1, mask); - andr(tmp, old, not_mask); - orr(tmp, tmp, new_val); - sc_w(tmp, tmp, aligned_addr, release); - bnez(tmp, fail); + lr_w(result, aligned_addr, acquire); + andr(scratch0, result, mask); + bne(scratch0, expected, fail); + + andr(scratch0, result, scratch1); // scratch1 is ~mask + orr(scratch0, scratch0, new_val); + sc_w(scratch0, scratch0, aligned_addr, release); + bnez(scratch0, fail); } // Success diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp index 568df056222..54f7127106b 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp @@ -1146,10 +1146,9 @@ public: enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release, Register result); - void cmpxchg_narrow_value_helper(Register addr, Register expected, - Register new_val, + void cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val, enum operand_size size, - Register tmp1, Register tmp2, Register tmp3); + Register shift, Register mask, Register aligned_addr); void cmpxchg_narrow_value(Register addr, Register expected, Register new_val, enum operand_size size,