stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8256011: Shenandoah: Don't resurrect finalizably reachable objects

Browse Source 
Reviewed-by: shade, zgu
This commit is contained in:
Roman Kennke 2020-11-13 09:47:00 +00:00
parent 41139e31c0
commit b0c28fadaa
16 changed files with 303 additions and 283 deletions

2
src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp

@ -109,7 +109,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
__ xchg(access.resolved_addr(), value_opr, result, tmp);
if (access.is_oop()) {
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), ShenandoahBarrierSet::AccessKind::NORMAL);
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators());
LIR_Opr tmp = gen->new_register(type);
__ move(result, tmp);
result = tmp;

144
src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@ -225,11 +225,17 @@ void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssemb
}
}
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, ShenandoahBarrierSet::AccessKind kind) {
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");
assert(dst != rscratch2, "need rscratch2");
assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
bool is_narrow = UseCompressedOops && !is_native;
Label heap_stable, not_cset;
__ enter();
Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
@ -252,7 +258,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ mov(r0, dst);
// Test for in-cset
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
if (is_strong) {
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
__ ldrb(rscratch2, Address(rscratch2, rscratch1));
@ -260,26 +266,22 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
}
__ push_call_clobbered_registers();
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
if (is_strong) {
if (is_narrow) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
}
} else if (is_weak) {
if (is_narrow) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
break;
default:
ShouldNotReachHere();
}
} else {
assert(is_phantom, "only remaining strength");
assert(!is_narrow, "phantom access cannot be narrow");
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
}
__ blr(lr);
__ mov(rscratch1, r0);
@ -338,8 +340,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
load_reference_barrier(masm, dst, src, kind);
load_reference_barrier(masm, dst, src, decorators);
if (dst != result_dst) {
__ mov(result_dst, dst);
@ -617,6 +618,12 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
__ bind(*stub->entry());
DecoratorSet decorators = stub->decorators();
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
Register addr = stub->addr()->as_pointer_register();
@ -629,42 +636,27 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
__ mov(res, obj);
}
// Check for null.
__ cbz(res, *stub->continuation());
if (is_strong) {
// Check for object in cset.
__ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
__ ldrb(tmp2, Address(tmp2, tmp1));
__ cbz(tmp2, *stub->continuation());
}
// Check for object in cset.
__ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
__ ldrb(tmp2, Address(tmp2, tmp1));
__ cbz(tmp2, *stub->continuation());
// Check if object is already forwarded.
Label slow_path;
__ ldr(tmp1, Address(res, oopDesc::mark_offset_in_bytes()));
__ eon(tmp1, tmp1, zr);
__ ands(zr, tmp1, markWord::lock_mask_in_place);
__ br(Assembler::NE, slow_path);
// Decode forwarded object.
__ orr(tmp1, tmp1, markWord::marked_value);
__ eon(res, tmp1, zr);
__ b(*stub->continuation());
__ bind(slow_path);
ce->store_parameter(res, 0);
ce->store_parameter(addr, 1);
switch (stub->kind()) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
__ far_call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
__ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ far_call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin()));
break;
default:
ShouldNotReachHere();
if (is_strong) {
if (is_native) {
__ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
} else {
__ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
}
} else if (is_weak) {
__ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
} else {
assert(is_phantom, "only remaining strength");
__ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
}
__ b(*stub->continuation());
@ -720,33 +712,39 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss
__ epilogue();
}
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) {
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
__ prologue("shenandoah_load_reference_barrier", false);
// arg0 : object to be resolved
__ push_call_clobbered_registers();
__ load_parameter(0, r0);
__ load_parameter(1, r1);
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
if (is_strong) {
if (is_native) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
} else {
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
}
} else if (is_weak) {
assert(!is_native, "weak must not be called off-heap");
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
break;
default:
ShouldNotReachHere();
}
} else {
assert(is_phantom, "only remaining strength");
assert(is_native, "phantom must only be called off-heap");
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
}
__ blr(lr);
__ mov(rscratch1, r0);

4
src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp

@ -56,7 +56,7 @@ private:
void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, ShenandoahBarrierSet::AccessKind kind);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators);
public:
@ -66,7 +66,7 @@ public:
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators);
#endif
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,

3
src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp

@ -111,8 +111,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
__ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);
if (access.is_oop()) {
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(access.decorators(), access.type());
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), kind);
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), access.decorators());
LIR_Opr tmp = gen->new_register(type);
__ move(result, tmp);
result = tmp;

165
src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

@ -268,9 +268,15 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
__ bind(done);
}
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind) {
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
bool is_narrow = UseCompressedOops && !is_native;
Label heap_stable, not_cset;
__ block_comment("load_reference_barrier { ");
@ -292,7 +298,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ jcc(Assembler::zero, heap_stable);
Register tmp1 = noreg, tmp2 = noreg;
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
if (is_strong) {
// Test for object in cset
// Allocate temporary registers
for (int i = 0; i < 8; i++) {
@ -357,26 +363,22 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
}
save_xmm_registers(masm);
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), arg0, arg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), arg0, arg1);
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1);
} else {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1);
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1);
break;
default:
ShouldNotReachHere();
if (is_strong) {
if (is_narrow) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), arg0, arg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), arg0, arg1);
}
} else if (is_weak) {
if (is_narrow) {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), arg0, arg1);
} else {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), arg0, arg1);
}
} else {
assert(is_phantom, "only remaining strength");
assert(!is_narrow, "phantom access cannot be narrow");
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), arg0, arg1);
}
restore_xmm_registers(masm);
@ -401,7 +403,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ bind(not_cset);
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
if (is_strong) {
__ pop(tmp2);
__ pop(tmp1);
}
@ -499,8 +501,7 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
load_reference_barrier(masm, dst, src, kind);
load_reference_barrier(masm, dst, src, decorators);
// Move loaded oop to final destination
if (dst != result_dst) {
@ -813,6 +814,12 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
__ bind(*stub->entry());
DecoratorSet decorators = stub->decorators();
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
Register obj = stub->obj()->as_register();
Register res = stub->result()->as_register();
Register addr = stub->addr()->as_pointer_register();
@ -828,40 +835,37 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
__ mov(res, obj);
}
// Check for null.
__ testptr(res, res);
__ jcc(Assembler::zero, *stub->continuation());
// Check for object being in the collection set.
__ mov(tmp1, res);
__ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
__ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
if (is_strong) {
// Check for object being in the collection set.
__ mov(tmp1, res);
__ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
__ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
#ifdef _LP64
__ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
__ testbool(tmp2);
__ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
__ testbool(tmp2);
#else
// On x86_32, C1 register allocator can give us the register without 8-bit support.
// Do the full-register access and test to avoid compilation failures.
__ movptr(tmp2, Address(tmp2, tmp1, Address::times_1));
__ testptr(tmp2, 0xFF);
// On x86_32, C1 register allocator can give us the register without 8-bit support.
// Do the full-register access and test to avoid compilation failures.
__ movptr(tmp2, Address(tmp2, tmp1, Address::times_1));
__ testptr(tmp2, 0xFF);
#endif
__ jcc(Assembler::zero, *stub->continuation());
__ jcc(Assembler::zero, *stub->continuation());
}
__ bind(slow_path);
ce->store_parameter(res, 0);
ce->store_parameter(addr, 1);
switch (stub->kind()) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
__ call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
__ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin()));
break;
default:
ShouldNotReachHere();
if (is_strong) {
if (is_native) {
__ call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
} else {
__ call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
}
} else if (is_weak) {
__ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
} else {
assert(is_phantom, "only remaining strength");
__ call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
}
__ jmp(*stub->continuation());
}
@ -926,49 +930,52 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss
__ epilogue();
}
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) {
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
__ prologue("shenandoah_load_reference_barrier", false);
// arg0 : object to be resolved
__ save_live_registers_no_oop_map(true);
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
#ifdef _LP64
__ load_parameter(0, c_rarg0);
__ load_parameter(1, c_rarg1);
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (is_strong) {
if (is_native) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1);
} else {
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), c_rarg0, c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), c_rarg0, c_rarg1);
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
}
} else if (is_weak) {
assert(!is_native, "weak must not be called off-heap");
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
break;
default:
ShouldNotReachHere();
}
} else {
assert(is_phantom, "only remaining strength");
assert(is_native, "phantom must only be called off-heap");
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
}
#else
__ load_parameter(0, rax);
__ load_parameter(1, rbx);
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx);
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx);
break;
default:
ShouldNotReachHere();
if (is_strong) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), rax, rbx);
} else if (is_weak) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx);
} else {
assert(is_phantom, "only remaining strength");
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), rax, rbx);
}
#endif

4
src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp

@ -64,10 +64,10 @@ public:
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators);
#endif
void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, DecoratorSet decorators);
void cmpxchg_oop(MacroAssembler* masm,
Register res, Address addr, Register oldval, Register newval,

51
src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp

@ -51,9 +51,10 @@ void ShenandoahLoadReferenceBarrierStub::emit_code(LIR_Assembler* ce) {
ShenandoahBarrierSetC1::ShenandoahBarrierSetC1() :
_pre_barrier_c1_runtime_code_blob(NULL),
_load_reference_barrier_normal_rt_code_blob(NULL),
_load_reference_barrier_native_rt_code_blob(NULL),
_load_reference_barrier_weak_rt_code_blob(NULL) {}
_load_reference_barrier_strong_rt_code_blob(NULL),
_load_reference_barrier_strong_native_rt_code_blob(NULL),
_load_reference_barrier_weak_rt_code_blob(NULL),
_load_reference_barrier_phantom_rt_code_blob(NULL) {}
void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
// First we test whether marking is in progress.
@ -109,15 +110,15 @@ void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info,
__ branch_destination(slow->continuation());
}
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind) {
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators) {
if (ShenandoahLoadRefBarrier) {
return load_reference_barrier_impl(gen, obj, addr, kind);
return load_reference_barrier_impl(gen, obj, addr, decorators);
} else {
return obj;
}
}
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind) {
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");
obj = ensure_in_register(gen, obj, T_OBJECT);
@ -150,7 +151,7 @@ LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, L
}
__ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, kind);
CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, decorators);
__ branch(lir_cond_notEqual, slow);
__ branch_destination(slow->continuation());
@ -213,8 +214,7 @@ void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result)
if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
LIR_Opr tmp = gen->new_register(T_OBJECT);
BarrierSetC1::load_at_resolved(access, tmp);
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
tmp = load_reference_barrier(gen, tmp, access.resolved_addr(), kind);
tmp = load_reference_barrier(gen, tmp, access.resolved_addr(), decorators);
__ move(tmp, result);
} else {
BarrierSetC1::load_at_resolved(access, result);
@ -253,14 +253,14 @@ class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure
class C1ShenandoahLoadReferenceBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
private:
const ShenandoahBarrierSet::AccessKind _kind;
const DecoratorSet _decorators;
public:
C1ShenandoahLoadReferenceBarrierCodeGenClosure(ShenandoahBarrierSet::AccessKind kind) : _kind(kind) {}
C1ShenandoahLoadReferenceBarrierCodeGenClosure(DecoratorSet decorators) : _decorators(decorators) {}
virtual OopMapSet* generate_code(StubAssembler* sasm) {
ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
bs->generate_c1_load_reference_barrier_runtime_stub(sasm, _kind);
bs->generate_c1_load_reference_barrier_runtime_stub(sasm, _decorators);
return NULL;
}
};
@ -271,19 +271,24 @@ void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob)
"shenandoah_pre_barrier_slow",
false, &pre_code_gen_cl);
if (ShenandoahLoadRefBarrier) {
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_code_gen_cl(ShenandoahBarrierSet::AccessKind::NORMAL);
_load_reference_barrier_normal_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_slow",
false, &lrb_code_gen_cl);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_strong_code_gen_cl(ON_STRONG_OOP_REF);
_load_reference_barrier_strong_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_strong_slow",
false, &lrb_strong_code_gen_cl);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_native_code_gen_cl(ShenandoahBarrierSet::AccessKind::NATIVE);
_load_reference_barrier_native_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_native_slow",
false, &lrb_native_code_gen_cl);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_strong_native_code_gen_cl(ON_STRONG_OOP_REF | IN_NATIVE);
_load_reference_barrier_strong_native_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_strong_native_slow",
false, &lrb_strong_native_code_gen_cl);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_weak_code_gen_cl(ShenandoahBarrierSet::AccessKind::WEAK);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_weak_code_gen_cl(ON_WEAK_OOP_REF);
_load_reference_barrier_weak_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_weak_slow",
false, &lrb_weak_code_gen_cl);
"shenandoah_load_reference_barrier_weak_slow",
false, &lrb_weak_code_gen_cl);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_phantom_code_gen_cl(ON_PHANTOM_OOP_REF | IN_NATIVE);
_load_reference_barrier_phantom_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_phantom_slow",
false, &lrb_phantom_code_gen_cl);
}
}

35
src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp

@ -94,10 +94,10 @@ private:
LIR_Opr _result;
LIR_Opr _tmp1;
LIR_Opr _tmp2;
ShenandoahBarrierSet::AccessKind _kind;
DecoratorSet _decorators;
public:
ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, ShenandoahBarrierSet::AccessKind kind) :
_obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2), _kind(kind)
ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, DecoratorSet decorators) :
_obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2), _decorators(decorators)
{
assert(_obj->is_register(), "should be register");
assert(_addr->is_register(), "should be register");
@ -111,7 +111,7 @@ public:
LIR_Opr result() const { return _result; }
LIR_Opr tmp1() const { return _tmp1; }
LIR_Opr tmp2() const { return _tmp2; }
ShenandoahBarrierSet::AccessKind kind() const { return _kind; }
DecoratorSet decorators() const { return _decorators; }
virtual void emit_code(LIR_Assembler* e);
virtual void visit(LIR_OpVisitState* visitor) {
@ -190,16 +190,17 @@ public:
class ShenandoahBarrierSetC1 : public BarrierSetC1 {
private:
CodeBlob* _pre_barrier_c1_runtime_code_blob;
CodeBlob* _load_reference_barrier_normal_rt_code_blob;
CodeBlob* _load_reference_barrier_native_rt_code_blob;
CodeBlob* _load_reference_barrier_strong_rt_code_blob;
CodeBlob* _load_reference_barrier_strong_native_rt_code_blob;
CodeBlob* _load_reference_barrier_weak_rt_code_blob;
CodeBlob* _load_reference_barrier_phantom_rt_code_blob;
void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val);
LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind);
LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators);
LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators);
LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind);
LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, DecoratorSet decorators);
LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj, BasicType type);
@ -211,20 +212,26 @@ public:
return _pre_barrier_c1_runtime_code_blob;
}
CodeBlob* load_reference_barrier_normal_rt_code_blob() {
assert(_load_reference_barrier_normal_rt_code_blob != NULL, "");
return _load_reference_barrier_normal_rt_code_blob;
CodeBlob* load_reference_barrier_strong_rt_code_blob() {
assert(_load_reference_barrier_strong_rt_code_blob != NULL, "");
return _load_reference_barrier_strong_rt_code_blob;
}
CodeBlob* load_reference_barrier_native_rt_code_blob() {
assert(_load_reference_barrier_native_rt_code_blob != NULL, "");
return _load_reference_barrier_native_rt_code_blob;
CodeBlob* load_reference_barrier_strong_native_rt_code_blob() {
assert(_load_reference_barrier_strong_native_rt_code_blob != NULL, "");
return _load_reference_barrier_strong_native_rt_code_blob;
}
CodeBlob* load_reference_barrier_weak_rt_code_blob() {
assert(_load_reference_barrier_weak_rt_code_blob != NULL, "");
return _load_reference_barrier_weak_rt_code_blob;
}
CodeBlob* load_reference_barrier_phantom_rt_code_blob() {
assert(_load_reference_barrier_phantom_rt_code_blob != NULL, "");
return _load_reference_barrier_phantom_rt_code_blob;
}
protected:
virtual void store_at_resolved(LIRAccess& access, LIR_Opr value);

22
src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

@ -303,10 +303,11 @@ bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
}
address entry_point = call->as_CallLeaf()->entry_point();
return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)) ||
return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
}
bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
@ -546,8 +547,7 @@ Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val
// 2: apply LRB if needed
if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
load = new ShenandoahLoadReferenceBarrierNode(NULL, load, kind);
load = new ShenandoahLoadReferenceBarrierNode(NULL, load, decorators);
if (access.is_parse_access()) {
load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
} else {
@ -644,8 +644,7 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess
load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
}
#endif
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(access.decorators(), access.type());
load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, kind));
load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, access.decorators()));
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
@ -713,8 +712,7 @@ Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& acces
}
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
if (access.is_oop()) {
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(access.decorators(), access.type());
result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, kind));
result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, access.decorators()));
shenandoah_write_barrier_pre(kit, false /* do_load */,
NULL, NULL, max_juint, NULL, NULL,
result /* pre_val */, T_OBJECT);
@ -1062,15 +1060,15 @@ Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_resh
Node* in1 = n->in(1);
Node* in2 = n->in(2);
// If one input is NULL, then step over the barriers normal LRB barriers on the other input
// If one input is NULL, then step over the strong LRB barriers on the other input
if (in1->bottom_type() == TypePtr::NULL_PTR &&
!((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
((ShenandoahLoadReferenceBarrierNode*)in2)->kind() != ShenandoahBarrierSet::AccessKind::NORMAL)) {
!ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in2)->decorators()))) {
in2 = step_over_gc_barrier(in2);
}
if (in2->bottom_type() == TypePtr::NULL_PTR &&
!((in1->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
((ShenandoahLoadReferenceBarrierNode*)in1)->kind() != ShenandoahBarrierSet::AccessKind::NORMAL)) {
!ShenandoahBarrierSet::is_strong_access(((ShenandoahLoadReferenceBarrierNode*)in1)->decorators()))) {
in1 = step_over_gc_barrier(in1);
}

79
src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

@ -960,7 +960,7 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl,
}
void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem,
ShenandoahBarrierSet::AccessKind kind, PhaseIdealLoop* phase) {
DecoratorSet decorators, PhaseIdealLoop* phase) {
IdealLoopTree*loop = phase->get_loop(ctrl);
const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr();
@ -973,25 +973,32 @@ void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* lo
address calladdr = NULL;
const char* name = NULL;
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NATIVE:
bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
bool is_narrow = UseCompressedOops && !is_native;
if (is_strong) {
if (is_narrow) {
calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
name = "load_reference_barrier_strong_narrow";
} else {
calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
name = "load_reference_barrier_strong";
}
} else if (is_weak) {
if (is_narrow) {
calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
name = "load_reference_barrier_weak_narrow";
} else {
calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
name = "load_reference_barrier_native";
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
calladdr = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow) :
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
name = "load_reference_barrier_weak";
break;
case ShenandoahBarrierSet::AccessKind::NORMAL:
calladdr = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) :
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
name = "load_reference_barrier";
break;
default:
ShouldNotReachHere();
}
} else {
assert(is_phantom, "only remaining strength");
assert(!is_narrow, "phantom access cannot be narrow");
calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
name = "load_reference_barrier_phantom";
}
Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
@ -1357,7 +1364,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
// even for non-cset objects to prevent ressurrection of such objects.
// Wires !in_cset(obj) to slot 2 of region and phis
Node* not_cset_ctrl = NULL;
if (lrb->kind() == ShenandoahBarrierSet::AccessKind::NORMAL) {
if (ShenandoahBarrierSet::is_strong_access(lrb->decorators())) {
test_in_cset(ctrl, not_cset_ctrl, val, raw_mem, phase);
}
if (not_cset_ctrl != NULL) {
@ -1408,7 +1415,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
}
}
}
call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->kind(), phase);
call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->decorators(), phase);
region->init_req(_evac_path, ctrl);
val_phi->init_req(_evac_path, val);
raw_mem_phi->init_req(_evac_path, result_mem);
@ -2904,40 +2911,32 @@ void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_p
}
}
ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, ShenandoahBarrierSet::AccessKind kind)
: Node(ctrl, obj), _kind(kind) {
ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, DecoratorSet decorators)
: Node(ctrl, obj), _decorators(decorators) {
ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this);
}
ShenandoahBarrierSet::AccessKind ShenandoahLoadReferenceBarrierNode::kind() const {
return _kind;
DecoratorSet ShenandoahLoadReferenceBarrierNode::decorators() const {
return _decorators;
}
uint ShenandoahLoadReferenceBarrierNode::size_of() const {
return sizeof(*this);
}
static DecoratorSet mask_decorators(DecoratorSet decorators) {
return decorators & (ON_STRONG_OOP_REF | ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF | IN_NATIVE);
}
uint ShenandoahLoadReferenceBarrierNode::hash() const {
uint hash = Node::hash();
switch (_kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
hash += 0;
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
hash += 1;
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
hash += 2;
break;
default:
ShouldNotReachHere();
}
hash += mask_decorators(_decorators);
return hash;
}
bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const {
return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier &&
_kind == ((const ShenandoahLoadReferenceBarrierNode&)n)._kind;
mask_decorators(_decorators) == mask_decorators(((const ShenandoahLoadReferenceBarrierNode&)n)._decorators);
}
const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
@ -2949,7 +2948,7 @@ const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {
return t;
}
if (kind() == ShenandoahBarrierSet::AccessKind::NORMAL) {
if (ShenandoahBarrierSet::is_strong_access(decorators())) {
return t;
}
@ -2965,7 +2964,7 @@ const Type* ShenandoahLoadReferenceBarrierNode::Value(PhaseGVN* phase) const {
return t2;
}
if (kind() == ShenandoahBarrierSet::AccessKind::NORMAL) {
if (ShenandoahBarrierSet::is_strong_access(decorators())) {
return t2;
}

8
src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp

@ -62,7 +62,7 @@ private:
static void test_gc_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase, int flags);
static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem,
ShenandoahBarrierSet::AccessKind kind, PhaseIdealLoop* phase);
DecoratorSet decorators, PhaseIdealLoop* phase);
static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase);
static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase);
static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase);
@ -231,12 +231,12 @@ public:
};
private:
ShenandoahBarrierSet::AccessKind _kind;
DecoratorSet _decorators;
public:
ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val, ShenandoahBarrierSet::AccessKind kind);
ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val, DecoratorSet decorators);
ShenandoahBarrierSet::AccessKind kind() const;
DecoratorSet decorators() const;
virtual int Opcode() const;
virtual const Type* bottom_type() const;
virtual const Type* Value(PhaseGVN* phase) const;

10
src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@ -98,16 +98,6 @@ bool ShenandoahBarrierSet::need_keep_alive_barrier(DecoratorSet decorators,Basic
return (on_weak_ref || unknown) && keep_alive;
}
ShenandoahBarrierSet::AccessKind ShenandoahBarrierSet::access_kind(DecoratorSet decorators, BasicType type) {
if ((decorators & IN_NATIVE) != 0) {
return AccessKind::NATIVE;
} else if ((decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0) {
return AccessKind::WEAK;
} else {
return AccessKind::NORMAL;
}
}
void ShenandoahBarrierSet::on_thread_create(Thread* thread) {
// Create thread local data
ShenandoahThreadLocalData::create(thread);

29
src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp

@ -33,18 +33,6 @@
class ShenandoahBarrierSetAssembler;
class ShenandoahBarrierSet: public BarrierSet {
public:
enum class AccessKind {
// Regular in-heap access on reference fields
NORMAL,
// Off-heap reference access
NATIVE,
// In-heap reference access on referent fields of j.l.r.Reference objects
WEAK
};
private:
ShenandoahHeap* _heap;
BufferNode::Allocator _satb_mark_queue_buffer_allocator;
@ -65,7 +53,22 @@ public:
static bool need_load_reference_barrier(DecoratorSet decorators, BasicType type);
static bool need_keep_alive_barrier(DecoratorSet decorators, BasicType type);
static AccessKind access_kind(DecoratorSet decorators, BasicType type);
static bool is_strong_access(DecoratorSet decorators) {
return (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF)) == 0;
}
static bool is_weak_access(DecoratorSet decorators) {
return (decorators & (ON_WEAK_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0;
}
static bool is_phantom_access(DecoratorSet decorators) {
return (decorators & ON_PHANTOM_OOP_REF) != 0;
}
static bool is_native_access(DecoratorSet decorators) {
return (decorators & IN_NATIVE) != 0;
}
void print_on(outputStream* st) const;

12
src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@ -102,13 +102,21 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj) {
template <DecoratorSet decorators, class T>
inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj, T* load_addr) {
// Prevent resurrection of unreachable non-strorg references.
if (!HasDecorator<decorators, ON_STRONG_OOP_REF>::value && obj != NULL &&
// Prevent resurrection of unreachable phantom (i.e. weak-native) references.
if (HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value && obj != NULL &&
_heap->is_concurrent_weak_root_in_progress() &&
!_heap->marking_context()->is_marked(obj)) {
return NULL;
}
// Prevent resurrection of unreachable weak references.
if ((HasDecorator<decorators, ON_WEAK_OOP_REF>::value || HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) &&
obj != NULL && _heap->is_concurrent_weak_root_in_progress() &&
!_heap->marking_context()->is_marked_strong(obj)) {
assert(Thread::current()->is_Java_thread(), "only Java threads get here");
return NULL;
}
// Prevent resurrection of unreachable objects that are visited during
// concurrent class-unloading.
if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value && obj != NULL &&

12
src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp

@ -50,11 +50,11 @@ JRT_LEAF(void, ShenandoahRuntime::write_ref_field_pre_entry(oopDesc* orig, JavaT
ShenandoahThreadLocalData::satb_mark_queue(thread).enqueue_known_active(orig);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier(oopDesc* src, oop* load_addr))
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_strong(oopDesc* src, oop* load_addr))
return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_narrow(oopDesc* src, narrowOop* load_addr))
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_strong_narrow(oopDesc* src, narrowOop* load_addr))
return ShenandoahBarrierSet::barrier_set()->load_reference_barrier_mutator(src, load_addr);
JRT_END
@ -67,9 +67,13 @@ JRT_LEAF(void, ShenandoahRuntime::shenandoah_clone_barrier(oopDesc* src))
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak(oopDesc * src, oop* load_addr))
return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<ON_UNKNOWN_OOP_REF, oop>(oop(src), load_addr);
return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<ON_WEAK_OOP_REF, oop>(oop(src), load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak_narrow(oopDesc * src, narrowOop* load_addr))
return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<ON_UNKNOWN_OOP_REF, narrowOop>(oop(src), load_addr);
return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<ON_WEAK_OOP_REF, narrowOop>(oop(src), load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_phantom(oopDesc * src, oop* load_addr))
return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<ON_PHANTOM_OOP_REF, oop>(oop(src), load_addr);
JRT_END

6
src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp

@ -38,12 +38,14 @@ public:
static void write_ref_field_pre_entry(oopDesc* orig, JavaThread* thread);
static oopDesc* load_reference_barrier(oopDesc* src, oop* load_addr);
static oopDesc* load_reference_barrier_narrow(oopDesc* src, narrowOop* load_addr);
static oopDesc* load_reference_barrier_strong(oopDesc* src, oop* load_addr);
static oopDesc* load_reference_barrier_strong_narrow(oopDesc* src, narrowOop* load_addr);
static oopDesc* load_reference_barrier_weak(oopDesc* src, oop* load_addr);
static oopDesc* load_reference_barrier_weak_narrow(oopDesc* src, narrowOop* load_addr);
static oopDesc* load_reference_barrier_phantom(oopDesc* src, oop* load_addr);
static void shenandoah_clone_barrier(oopDesc* src);
};