diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp index 738511c6f50..9913a589683 100644 --- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp @@ -2600,6 +2600,13 @@ void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { Unimplemented(); } + // There might be a volatile load before this Unsafe CAS. + if (support_IRIW_for_not_multiple_copy_atomic_cpu) { + __ sync(); + } else { + __ lwsync(); + } + if (is_64bit) { __ cmpxchgd(BOOL_RESULT, /*current_value=*/R0, cmp_value, new_value, addr, MacroAssembler::MemBarNone, @@ -2961,9 +2968,24 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr assert(addr->disp() == 0 && addr->index()->is_illegal(), "use leal!"); const Register Rptr = addr->base()->as_pointer_register(), Rtmp = tmp->as_register(); - Register Rco = noreg; - if (UseCompressedOops && data->is_oop()) { - Rco = __ encode_heap_oop(Rtmp, data->as_register()); + Register Robj = noreg; + if (data->is_oop()) { + if (UseCompressedOops) { + Robj = __ encode_heap_oop(Rtmp, data->as_register()); + } else { + Robj = data->as_register(); + if (Robj == dest->as_register()) { // May happen with ZGC. + __ mr(Rtmp, Robj); + Robj = Rtmp; + } + } + } + + // There might be a volatile load before this Unsafe OP. + if (support_IRIW_for_not_multiple_copy_atomic_cpu) { + __ sync(); + } else { + __ lwsync(); } Label Lretry; @@ -2983,18 +3005,11 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr } else if (data->is_oop()) { assert(code == lir_xchg, "xadd for oops"); const Register Rold = dest->as_register(); + assert_different_registers(Rptr, Rold, Robj); if (UseCompressedOops) { - assert_different_registers(Rptr, Rold, Rco); __ lwarx(Rold, Rptr, MacroAssembler::cmpxchgx_hint_atomic_update()); - __ stwcx_(Rco, Rptr); + __ stwcx_(Robj, Rptr); } else { - Register Robj = data->as_register(); - assert_different_registers(Rptr, Rold, Rtmp); - assert_different_registers(Rptr, Robj, Rtmp); - if (Robj == Rold) { // May happen with ZGC. - __ mr(Rtmp, Robj); - Robj = Rtmp; - } __ ldarx(Rold, Rptr, MacroAssembler::cmpxchgx_hint_atomic_update()); __ stdcx_(Robj, Rptr); } @@ -3022,6 +3037,12 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr if (UseCompressedOops && data->is_oop()) { __ decode_heap_oop(dest->as_register()); } + + if (support_IRIW_for_not_multiple_copy_atomic_cpu) { + __ isync(); + } else { + __ sync(); + } } diff --git a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp index ecc40d6fde7..32aab91c7d3 100644 --- a/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp +++ b/src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp @@ -639,13 +639,6 @@ LIR_Opr LIRGenerator::atomic_cmpxchg(BasicType type, LIR_Opr addr, LIRItem& cmp_ cmp_value.load_item(); new_value.load_item(); - // Volatile load may be followed by Unsafe CAS. - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar(); - } else { - __ membar_release(); - } - if (is_reference_type(type)) { if (UseCompressedOops) { t1 = new_register(T_OBJECT); @@ -670,21 +663,7 @@ LIR_Opr LIRGenerator::atomic_xchg(BasicType type, LIR_Opr addr, LIRItem& value) LIR_Opr tmp = FrameMap::R0_opr; value.load_item(); - - // Volatile load may be followed by Unsafe CAS. - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar(); - } else { - __ membar_release(); - } - __ xchg(addr, value.result(), result, tmp); - - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar_acquire(); - } else { - __ membar(); - } return result; } @@ -694,21 +673,7 @@ LIR_Opr LIRGenerator::atomic_add(BasicType type, LIR_Opr addr, LIRItem& value) { LIR_Opr tmp = FrameMap::R0_opr; value.load_item(); - - // Volatile load may be followed by Unsafe CAS. - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar(); // To be safe. Unsafe semantics are unclear. - } else { - __ membar_release(); - } - __ xadd(addr, value.result(), result, tmp); - - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar_acquire(); - } else { - __ membar(); - } return result; } diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp index fc06e1b71e0..6d9a1db1ed4 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * Copyright (c) 2012, 2021 SAP SE. All rights reserved. + * Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved. + * Copyright (c) 2012, 2023 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -53,8 +53,13 @@ void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler *masm) { __ encode_heap_oop(new_val, new_val); } - // Due to the memory barriers emitted in ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved, - // there is no need to specify stronger memory semantics. + // There might be a volatile load before this Unsafe CAS. + if (support_IRIW_for_not_multiple_copy_atomic_cpu) { + __ sync(); + } else { + __ lwsync(); + } + ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmp_val, new_val, tmp1, tmp2, false, result); @@ -63,6 +68,12 @@ void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler *masm) { __ decode_heap_oop(new_val); } + if (support_IRIW_for_not_multiple_copy_atomic_cpu) { + __ isync(); + } else { + __ sync(); + } + __ block_comment("} LIR_OpShenandoahCompareAndSwap (shenandaohgc)"); } @@ -80,14 +91,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI if (access.is_oop()) { LIRGenerator* gen = access.gen(); - if (ShenandoahCASBarrier) { - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar(); - } else { - __ membar_release(); - } - } - if (ShenandoahSATBBarrier) { pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(), LIR_OprFact::illegalOpr); @@ -104,12 +107,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result)); - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar_acquire(); - } else { - __ membar(); - } - return result; } } @@ -125,12 +122,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt value.load_item(); LIR_Opr value_opr = value.result(); - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar(); - } else { - __ membar_release(); - } - if (access.is_oop()) { value_opr = iu_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators()); } @@ -152,11 +143,5 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt } } - if (support_IRIW_for_not_multiple_copy_atomic_cpu) { - __ membar_acquire(); - } else { - __ membar(); - } - return result; } diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp index 1d07c6d573a..9ed47b688ff 100644 --- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp @@ -340,11 +340,12 @@ void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm, } __ ld(R0, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread); __ cmpxchgd(CCR0, tmp, (intptr_t)0, R0, ref_base, - MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update()); + MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), + noreg, need_restore ? nullptr : &slow_path); if (need_restore) { __ subf(ref_base, ind_or_offs, ref_base); + __ bne(CCR0, slow_path); } - __ bne(CCR0, slow_path); } else { // A non-atomic relocatable object won't get to the medium fast path due to a // raw null in the young generation. We only get here because the field is bad.