stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8320807: [PPC64][ZGC] C1 generates wrong code for atomics

Browse Source 
Reviewed-by: lucy, rrich
This commit is contained in:
Martin Doerr 2023-12-01 14:44:18 +00:00
parent 54957ac55f
commit 3087e14cde
4 changed files with 51 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
View File

@ -2600,6 +2600,13 @@ void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
Unimplemented();
}
// There might be a volatile load before this Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ sync();
} else {
__ lwsync();
}
if (is_64bit) {
__ cmpxchgd(BOOL_RESULT, /*current_value=*/R0, cmp_value, new_value, addr,
MacroAssembler::MemBarNone,
@ -2961,9 +2968,24 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr
assert(addr->disp() == 0 && addr->index()->is_illegal(), "use leal!");
const Register Rptr = addr->base()->as_pointer_register(),
Rtmp = tmp->as_register();
Register Rco = noreg;
if (UseCompressedOops && data->is_oop()) {
Rco = __ encode_heap_oop(Rtmp, data->as_register());
Register Robj = noreg;
if (data->is_oop()) {
if (UseCompressedOops) {
Robj = __ encode_heap_oop(Rtmp, data->as_register());
} else {
Robj = data->as_register();
if (Robj == dest->as_register()) { // May happen with ZGC.
__ mr(Rtmp, Robj);
Robj = Rtmp;
}
}
}
// There might be a volatile load before this Unsafe OP.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ sync();
} else {
__ lwsync();
}
Label Lretry;
@ -2983,18 +3005,11 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr
} else if (data->is_oop()) {
assert(code == lir_xchg, "xadd for oops");
const Register Rold = dest->as_register();
assert_different_registers(Rptr, Rold, Robj);
if (UseCompressedOops) {
assert_different_registers(Rptr, Rold, Rco);
__ lwarx(Rold, Rptr, MacroAssembler::cmpxchgx_hint_atomic_update());
__ stwcx_(Rco, Rptr);
__ stwcx_(Robj, Rptr);
} else {
Register Robj = data->as_register();
assert_different_registers(Rptr, Rold, Rtmp);
assert_different_registers(Rptr, Robj, Rtmp);
if (Robj == Rold) { // May happen with ZGC.
__ mr(Rtmp, Robj);
Robj = Rtmp;
}
__ ldarx(Rold, Rptr, MacroAssembler::cmpxchgx_hint_atomic_update());
__ stdcx_(Robj, Rptr);
}
@ -3022,6 +3037,12 @@ void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr
if (UseCompressedOops && data->is_oop()) {
__ decode_heap_oop(dest->as_register());
}
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ isync();
} else {
__ sync();
}
}

35
src/hotspot/cpu/ppc/c1_LIRGenerator_ppc.cpp
View File

@ -639,13 +639,6 @@ LIR_Opr LIRGenerator::atomic_cmpxchg(BasicType type, LIR_Opr addr, LIRItem& cmp_
cmp_value.load_item();
new_value.load_item();
// Volatile load may be followed by Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar();
} else {
__ membar_release();
}
if (is_reference_type(type)) {
if (UseCompressedOops) {
t1 = new_register(T_OBJECT);
@ -670,21 +663,7 @@ LIR_Opr LIRGenerator::atomic_xchg(BasicType type, LIR_Opr addr, LIRItem& value)
LIR_Opr tmp = FrameMap::R0_opr;
value.load_item();
// Volatile load may be followed by Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar();
} else {
__ membar_release();
}
__ xchg(addr, value.result(), result, tmp);
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
} else {
__ membar();
}
return result;
}
@ -694,21 +673,7 @@ LIR_Opr LIRGenerator::atomic_add(BasicType type, LIR_Opr addr, LIRItem& value) {
LIR_Opr tmp = FrameMap::R0_opr;
value.load_item();
// Volatile load may be followed by Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar(); // To be safe. Unsafe semantics are unclear.
} else {
__ membar_release();
}
__ xadd(addr, value.result(), result, tmp);
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
} else {
__ membar();
}
return result;
}

45
src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp
View File

@ -1,6 +1,6 @@
/*
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
* Copyright (c) 2012, 2021 SAP SE. All rights reserved.
* Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved.
* Copyright (c) 2012, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -53,8 +53,13 @@ void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler *masm) {
__ encode_heap_oop(new_val, new_val);
}
// Due to the memory barriers emitted in ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved,
// there is no need to specify stronger memory semantics.
// There might be a volatile load before this Unsafe CAS.
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ sync();
} else {
__ lwsync();
}
ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmp_val, new_val, tmp1, tmp2,
false, result);
@ -63,6 +68,12 @@ void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler *masm) {
__ decode_heap_oop(new_val);
}
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ isync();
} else {
__ sync();
}
__ block_comment("} LIR_OpShenandoahCompareAndSwap (shenandaohgc)");
}
@ -80,14 +91,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI
if (access.is_oop()) {
LIRGenerator* gen = access.gen();
if (ShenandoahCASBarrier) {
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar();
} else {
__ membar_release();
}
}
if (ShenandoahSATBBarrier) {
pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
LIR_OprFact::illegalOpr);
@ -104,12 +107,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI
__ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result));
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
} else {
__ membar();
}
return result;
}
}
@ -125,12 +122,6 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt
value.load_item();
LIR_Opr value_opr = value.result();
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar();
} else {
__ membar_release();
}
if (access.is_oop()) {
value_opr = iu_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators());
}
@ -152,11 +143,5 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt
}
}
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
__ membar_acquire();
} else {
__ membar();
}
return result;
}

5
src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
View File

@ -340,11 +340,12 @@ void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
}
__ ld(R0, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread);
__ cmpxchgd(CCR0, tmp, (intptr_t)0, R0, ref_base,
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update());
MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(),
noreg, need_restore ? nullptr : &slow_path);
if (need_restore) {
__ subf(ref_base, ind_or_offs, ref_base);
__ bne(CCR0, slow_path);
}
__ bne(CCR0, slow_path);
} else {
// A non-atomic relocatable object won't get to the medium fast path due to a
// raw null in the young generation. We only get here because the field is bad.