stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8254315: Shenandoah: Concurrent weak reference processing

Browse Source 
Reviewed-by: zgu, shade
This commit is contained in:
Roman Kennke 2020-11-03 18:58:46 +00:00
parent 83f3cf4298
commit f64a15d62e
55 changed files with 1864 additions and 802 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp
View File

@ -109,7 +109,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
__ xchg(access.resolved_addr(), value_opr, result, tmp);
if (access.is_oop()) {
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), false);
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), ShenandoahBarrierSet::AccessKind::NORMAL);
LIR_Opr tmp = gen->new_register(type);
__ move(result, tmp);
result = tmp;

80
src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
View File

@ -225,7 +225,7 @@ void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssemb
}
}
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, bool weak) {
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, ShenandoahBarrierSet::AccessKind kind) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");
assert(dst != rscratch2, "need rscratch2");
assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
@ -252,7 +252,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ mov(r0, dst);
// Test for in-cset
if (!weak) {
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
__ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
__ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
__ ldrb(rscratch2, Address(rscratch2, rscratch1));
@ -260,14 +260,26 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
}
__ push_call_clobbered_registers();
if (weak) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
} else {
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
}
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
break;
default:
ShouldNotReachHere();
}
__ blr(lr);
__ mov(rscratch1, r0);
@ -326,8 +338,8 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
bool weak = ShenandoahBarrierSet::use_load_reference_barrier_weak(decorators, type);
load_reference_barrier(masm, dst, src, weak);
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
load_reference_barrier(masm, dst, src, kind);
if (dst != result_dst) {
__ mov(result_dst, dst);
@ -641,10 +653,18 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
__ bind(slow_path);
ce->store_parameter(res, 0);
ce->store_parameter(addr, 1);
if (stub->is_weak()) {
__ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
} else {
__ far_call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
switch (stub->kind()) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
__ far_call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
__ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ far_call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin()));
break;
default:
ShouldNotReachHere();
}
__ b(*stub->continuation());
@ -700,19 +720,33 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss
__ epilogue();
}
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, bool is_weak) {
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) {
__ prologue("shenandoah_load_reference_barrier", false);
// arg0 : object to be resolved
__ push_call_clobbered_registers();
__ load_parameter(0, r0);
__ load_parameter(1, r1);
if (is_weak) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
} else if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier));
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
} else {
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
break;
default:
ShouldNotReachHere();
}
__ blr(lr);
__ mov(rscratch1, r0);

5
src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp
View File

@ -27,6 +27,7 @@
#include "asm/macroAssembler.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#ifdef COMPILER1
class LIR_Assembler;
class ShenandoahPreBarrierStub;
@ -55,7 +56,7 @@ private:
void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg);
void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, bool weak);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, ShenandoahBarrierSet::AccessKind kind);
public:
@ -65,7 +66,7 @@ public:
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, bool is_weak);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind);
#endif
virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,

2
src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp
View File

@ -111,7 +111,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt
__ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);
if (access.is_oop()) {
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), false);
result = load_reference_barrier(access.gen(), result, LIR_OprFact::addressConst(0), ShenandoahBarrierSet::AccessKind::NORMAL);
LIR_Opr tmp = gen->new_register(type);
__ move(result, tmp);
result = tmp;

96
src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
View File

@ -268,7 +268,7 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
__ bind(done);
}
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, bool weak) {
void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");
Label heap_stable, not_cset;
@ -292,7 +292,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ jcc(Assembler::zero, heap_stable);
Register tmp1 = noreg;
if (!weak) {
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
// Test for object in cset
// Allocate tmp-reg.
for (int i = 0; i < 8; i++) {
@ -338,14 +338,26 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ lea(tmp2, src);
save_xmm_registers(masm);
if (weak) {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), dst, tmp2);
} else {
if (UseCompressedOops) {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), dst, tmp2);
} else {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), dst, tmp2);
}
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (UseCompressedOops) {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), dst, tmp2);
} else {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), dst, tmp2);
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), dst, tmp2);
} else {
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), dst, tmp2);
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), dst, tmp2);
break;
default:
ShouldNotReachHere();
}
restore_xmm_registers(masm);
@ -370,7 +382,7 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
__ bind(not_cset);
if (!weak) {
if (kind == ShenandoahBarrierSet::AccessKind::NORMAL) {
__ pop(tmp1);
}
@ -467,8 +479,8 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
bool weak = ShenandoahBarrierSet::use_load_reference_barrier_weak(decorators, type);
load_reference_barrier(masm, dst, src, weak);
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
load_reference_barrier(masm, dst, src, kind);
// Move loaded oop to final destination
if (dst != result_dst) {
@ -818,10 +830,18 @@ void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assemble
__ bind(slow_path);
ce->store_parameter(res, 0);
ce->store_parameter(addr, 1);
if (stub->is_weak()) {
__ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
} else {
__ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
switch (stub->kind()) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
__ call(RuntimeAddress(bs->load_reference_barrier_normal_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
__ call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ call(RuntimeAddress(bs->load_reference_barrier_native_rt_code_blob()->code_begin()));
break;
default:
ShouldNotReachHere();
}
__ jmp(*stub->continuation());
}
@ -886,7 +906,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss
__ epilogue();
}
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, bool is_weak) {
void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind) {
__ prologue("shenandoah_load_reference_barrier", false);
// arg0 : object to be resolved
@ -895,20 +915,40 @@ void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_s
#ifdef _LP64
__ load_parameter(0, c_rarg0);
__ load_parameter(1, c_rarg1);
if (is_weak) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
} else if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
}
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
if (UseCompressedOops) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
}
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
break;
default:
ShouldNotReachHere();
}
#else
__ load_parameter(0, rax);
__ load_parameter(1, rbx);
if (is_weak) {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx);
} else {
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx);
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx);
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
case ShenandoahBarrierSet::AccessKind::NATIVE:
__ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), rax, rbx);
break;
default:
ShouldNotReachHere();
}
#endif

6
src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp
View File

@ -27,6 +27,8 @@
#include "asm/macroAssembler.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#ifdef COMPILER1
class LIR_Assembler;
class ShenandoahPreBarrierStub;
@ -62,10 +64,10 @@ public:
void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
void gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub);
void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, bool is_weak);
void generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, ShenandoahBarrierSet::AccessKind kind);
#endif
void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, bool weak);
void load_reference_barrier(MacroAssembler* masm, Register dst, Address src, ShenandoahBarrierSet::AccessKind kind);
void cmpxchg_oop(MacroAssembler* masm,
Register res, Address addr, Register oldval, Register newval,

37
src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
View File

@ -51,7 +51,9 @@ void ShenandoahLoadReferenceBarrierStub::emit_code(LIR_Assembler* ce) {
ShenandoahBarrierSetC1::ShenandoahBarrierSetC1() :
_pre_barrier_c1_runtime_code_blob(NULL),
_load_reference_barrier_rt_code_blob(NULL) {}
_load_reference_barrier_normal_rt_code_blob(NULL),
_load_reference_barrier_native_rt_code_blob(NULL),
_load_reference_barrier_weak_rt_code_blob(NULL) {}
void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
// First we test whether marking is in progress.
@ -107,15 +109,15 @@ void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info,
__ branch_destination(slow->continuation());
}
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, bool is_native) {
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind) {
if (ShenandoahLoadRefBarrier) {
return load_reference_barrier_impl(gen, obj, addr, is_native);
return load_reference_barrier_impl(gen, obj, addr, kind);
} else {
return obj;
}
}
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, bool is_native) {
LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind) {
assert(ShenandoahLoadRefBarrier, "Should be enabled");
obj = ensure_in_register(gen, obj, T_OBJECT);
@ -148,7 +150,7 @@ LIR_Opr ShenandoahBarrierSetC1::load_reference_barrier_impl(LIRGenerator* gen, L
}
__ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, is_native);
CodeStub* slow = new ShenandoahLoadReferenceBarrierStub(obj, addr, result, tmp1, tmp2, kind);
__ branch(lir_cond_notEqual, slow);
__ branch_destination(slow->continuation());
@ -211,8 +213,8 @@ void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result)
if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
LIR_Opr tmp = gen->new_register(T_OBJECT);
BarrierSetC1::load_at_resolved(access, tmp);
bool is_weak = ShenandoahBarrierSet::use_load_reference_barrier_weak(decorators, type);
tmp = load_reference_barrier(gen, tmp, access.resolved_addr(), is_weak);
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
tmp = load_reference_barrier(gen, tmp, access.resolved_addr(), kind);
__ move(tmp, result);
} else {
BarrierSetC1::load_at_resolved(access, result);
@ -251,14 +253,14 @@ class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure
class C1ShenandoahLoadReferenceBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
private:
const bool _is_weak;
const ShenandoahBarrierSet::AccessKind _kind;
public:
C1ShenandoahLoadReferenceBarrierCodeGenClosure(bool is_weak) : _is_weak(is_weak) {}
C1ShenandoahLoadReferenceBarrierCodeGenClosure(ShenandoahBarrierSet::AccessKind kind) : _kind(kind) {}
virtual OopMapSet* generate_code(StubAssembler* sasm) {
ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
bs->generate_c1_load_reference_barrier_runtime_stub(sasm, _is_weak);
bs->generate_c1_load_reference_barrier_runtime_stub(sasm, _kind);
return NULL;
}
};
@ -269,14 +271,19 @@ void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob)
"shenandoah_pre_barrier_slow",
false, &pre_code_gen_cl);
if (ShenandoahLoadRefBarrier) {
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_code_gen_cl(false);
_load_reference_barrier_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_code_gen_cl(ShenandoahBarrierSet::AccessKind::NORMAL);
_load_reference_barrier_normal_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_slow",
false, &lrb_code_gen_cl);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_weak_code_gen_cl(true);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_native_code_gen_cl(ShenandoahBarrierSet::AccessKind::NATIVE);
_load_reference_barrier_native_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_native_slow",
false, &lrb_native_code_gen_cl);
C1ShenandoahLoadReferenceBarrierCodeGenClosure lrb_weak_code_gen_cl(ShenandoahBarrierSet::AccessKind::WEAK);
_load_reference_barrier_weak_rt_code_blob = Runtime1::generate_blob(buffer_blob, -1,
"shenandoah_load_reference_barrier_weak_slow",
false, &lrb_weak_code_gen_cl);
"shenandoah_load_reference_barrier_weak_slow",
false, &lrb_weak_code_gen_cl);
}
}

26
src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp
View File

@ -94,10 +94,10 @@ private:
LIR_Opr _result;
LIR_Opr _tmp1;
LIR_Opr _tmp2;
bool _is_weak;
ShenandoahBarrierSet::AccessKind _kind;
public:
ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, bool is_weak) :
_obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2), _is_weak(is_weak)
ShenandoahLoadReferenceBarrierStub(LIR_Opr obj, LIR_Opr addr, LIR_Opr result, LIR_Opr tmp1, LIR_Opr tmp2, ShenandoahBarrierSet::AccessKind kind) :
_obj(obj), _addr(addr), _result(result), _tmp1(tmp1), _tmp2(tmp2), _kind(kind)
{
assert(_obj->is_register(), "should be register");
assert(_addr->is_register(), "should be register");
@ -111,7 +111,7 @@ public:
LIR_Opr result() const { return _result; }
LIR_Opr tmp1() const { return _tmp1; }
LIR_Opr tmp2() const { return _tmp2; }
bool is_weak() const { return _is_weak; }
ShenandoahBarrierSet::AccessKind kind() const { return _kind; }
virtual void emit_code(LIR_Assembler* e);
virtual void visit(LIR_OpVisitState* visitor) {
@ -190,15 +190,16 @@ public:
class ShenandoahBarrierSetC1 : public BarrierSetC1 {
private:
CodeBlob* _pre_barrier_c1_runtime_code_blob;
CodeBlob* _load_reference_barrier_rt_code_blob;
CodeBlob* _load_reference_barrier_normal_rt_code_blob;
CodeBlob* _load_reference_barrier_native_rt_code_blob;
CodeBlob* _load_reference_barrier_weak_rt_code_blob;
void pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val);
LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, bool is_native);
LIR_Opr load_reference_barrier(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind);
LIR_Opr storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators);
LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, bool is_native);
LIR_Opr load_reference_barrier_impl(LIRGenerator* gen, LIR_Opr obj, LIR_Opr addr, ShenandoahBarrierSet::AccessKind kind);
LIR_Opr ensure_in_register(LIRGenerator* gen, LIR_Opr obj, BasicType type);
@ -210,9 +211,14 @@ public:
return _pre_barrier_c1_runtime_code_blob;
}
CodeBlob* load_reference_barrier_rt_code_blob() {
assert(_load_reference_barrier_rt_code_blob != NULL, "");
return _load_reference_barrier_rt_code_blob;
CodeBlob* load_reference_barrier_normal_rt_code_blob() {
assert(_load_reference_barrier_normal_rt_code_blob != NULL, "");
return _load_reference_barrier_normal_rt_code_blob;
}
CodeBlob* load_reference_barrier_native_rt_code_blob() {
assert(_load_reference_barrier_native_rt_code_blob != NULL, "");
return _load_reference_barrier_native_rt_code_blob;
}
CodeBlob* load_reference_barrier_weak_rt_code_blob() {

18
src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
View File

@ -305,7 +305,8 @@ bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
address entry_point = call->as_CallLeaf()->entry_point();
return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
}
bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Node* n) {
@ -545,9 +546,8 @@ Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val
// 2: apply LRB if needed
if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
load = new ShenandoahLoadReferenceBarrierNode(NULL,
load,
ShenandoahBarrierSet::use_load_reference_barrier_weak(decorators, type));
ShenandoahBarrierSet::AccessKind kind = ShenandoahBarrierSet::access_kind(decorators, type);
load = new ShenandoahLoadReferenceBarrierNode(NULL, load, kind);
if (access.is_parse_access()) {
load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
} else {
@ -644,7 +644,7 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess
load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
}
#endif
load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, false));
load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, ShenandoahBarrierSet::AccessKind::NORMAL));
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
@ -712,7 +712,7 @@ Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& acces
}
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
if (access.is_oop()) {
result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, false));
result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, result, ShenandoahBarrierSet::AccessKind::NORMAL));
shenandoah_write_barrier_pre(kit, false /* do_load */,
NULL, NULL, max_juint, NULL, NULL,
result /* pre_val */, T_OBJECT);
@ -1060,15 +1060,15 @@ Node* ShenandoahBarrierSetC2::ideal_node(PhaseGVN* phase, Node* n, bool can_resh
Node* in1 = n->in(1);
Node* in2 = n->in(2);
// If one input is NULL, then step over the barriers (except LRB native) on the other input
// If one input is NULL, then step over the barriers normal LRB barriers on the other input
if (in1->bottom_type() == TypePtr::NULL_PTR &&
!((in2->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
((ShenandoahLoadReferenceBarrierNode*)in2)->is_weak())) {
((ShenandoahLoadReferenceBarrierNode*)in2)->kind() != ShenandoahBarrierSet::AccessKind::NORMAL)) {
in2 = step_over_gc_barrier(in2);
}
if (in2->bottom_type() == TypePtr::NULL_PTR &&
!((in1->Opcode() == Op_ShenandoahLoadReferenceBarrier) &&
((ShenandoahLoadReferenceBarrierNode*)in1)->is_weak())) {
((ShenandoahLoadReferenceBarrierNode*)in1)->kind() != ShenandoahBarrierSet::AccessKind::NORMAL)) {
in1 = step_over_gc_barrier(in1);
}

62
src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
View File

@ -956,7 +956,8 @@ void ShenandoahBarrierC2Support::test_in_cset(Node*& ctrl, Node*& not_cset_ctrl,
phase->register_new_node(cset_bool, old_ctrl);
}
void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_weak, PhaseIdealLoop* phase) {
void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem,
ShenandoahBarrierSet::AccessKind kind, PhaseIdealLoop* phase) {
IdealLoopTree*loop = phase->get_loop(ctrl);
const TypePtr* obj_type = phase->igvn().type(val)->is_oopptr();
@ -967,13 +968,28 @@ void ShenandoahBarrierC2Support::call_lrb_stub(Node*& ctrl, Node*& val, Node* lo
mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
phase->register_new_node(mm, ctrl);
address target = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) :
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
address calladdr = is_weak ? CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)
: target;
const char* name = is_weak ? "load_reference_barrier_native" : "load_reference_barrier";
address calladdr = NULL;
const char* name = NULL;
switch (kind) {
case ShenandoahBarrierSet::AccessKind::NATIVE:
calladdr = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
name = "load_reference_barrier_native";
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
calladdr = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow) :
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
name = "load_reference_barrier_weak";
break;
case ShenandoahBarrierSet::AccessKind::NORMAL:
calladdr = LP64_ONLY(UseCompressedOops) NOT_LP64(false) ?
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow) :
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier);
name = "load_reference_barrier";
break;
default:
ShouldNotReachHere();
}
Node* call = new CallLeafNode(ShenandoahBarrierSetC2::shenandoah_load_reference_barrier_Type(), calladdr, name, TypeRawPtr::BOTTOM);
call->init_req(TypeFunc::Control, ctrl);
@ -1338,7 +1354,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
// even for non-cset objects to prevent ressurrection of such objects.
// Wires !in_cset(obj) to slot 2 of region and phis
Node* not_cset_ctrl = NULL;
if (!lrb->is_weak()) {
if (lrb->kind() == ShenandoahBarrierSet::AccessKind::NORMAL) {
test_in_cset(ctrl, not_cset_ctrl, val, raw_mem, phase);
}
if (not_cset_ctrl != NULL) {
@ -1389,7 +1405,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
}
}
}
call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->is_weak(), phase);
call_lrb_stub(ctrl, val, addr, result_mem, raw_mem, lrb->kind(), phase);
region->init_req(_evac_path, ctrl);
val_phi->init_req(_evac_path, val);
raw_mem_phi->init_req(_evac_path, result_mem);
@ -2885,13 +2901,13 @@ void MemoryGraphFixer::fix_memory_uses(Node* mem, Node* replacement, Node* rep_p
}
}
ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, bool weak)
: Node(ctrl, obj), _weak(weak) {
ShenandoahLoadReferenceBarrierNode::ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* obj, ShenandoahBarrierSet::AccessKind kind)
: Node(ctrl, obj), _kind(kind) {
ShenandoahBarrierSetC2::bsc2()->state()->add_load_reference_barrier(this);
}
bool ShenandoahLoadReferenceBarrierNode::is_weak() const {
return _weak;
ShenandoahBarrierSet::AccessKind ShenandoahLoadReferenceBarrierNode::kind() const {
return _kind;
}
uint ShenandoahLoadReferenceBarrierNode::size_of() const {
@ -2899,12 +2915,26 @@ uint ShenandoahLoadReferenceBarrierNode::size_of() const {
}
uint ShenandoahLoadReferenceBarrierNode::hash() const {
return Node::hash() + (_weak ? 1 : 0);
uint hash = Node::hash();
switch (_kind) {
case ShenandoahBarrierSet::AccessKind::NORMAL:
hash += 0;
break;
case ShenandoahBarrierSet::AccessKind::WEAK:
hash += 1;
break;
case ShenandoahBarrierSet::AccessKind::NATIVE:
hash += 2;
break;
default:
ShouldNotReachHere();
}
return hash;
}
bool ShenandoahLoadReferenceBarrierNode::cmp( const Node &n ) const {
return Node::cmp(n) && n.Opcode() == Op_ShenandoahLoadReferenceBarrier &&
_weak == ((const ShenandoahLoadReferenceBarrierNode&)n)._weak;
_kind == ((const ShenandoahLoadReferenceBarrierNode&)n)._kind;
}
const Type* ShenandoahLoadReferenceBarrierNode::bottom_type() const {

10
src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
View File

@ -25,6 +25,7 @@
#ifndef SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP
#define SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "memory/allocation.hpp"
#include "opto/addnode.hpp"
#include "opto/graphKit.hpp"
@ -60,7 +61,8 @@ private:
static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase);
static void test_gc_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
PhaseIdealLoop* phase, int flags);
static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_weak, PhaseIdealLoop* phase);
static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem,
ShenandoahBarrierSet::AccessKind kind, PhaseIdealLoop* phase);
static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase);
static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase);
static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase);
@ -229,12 +231,12 @@ public:
};
private:
bool _weak;
ShenandoahBarrierSet::AccessKind _kind;
public:
ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val, bool native);
ShenandoahLoadReferenceBarrierNode(Node* ctrl, Node* val, ShenandoahBarrierSet::AccessKind kind);
bool is_weak() const;
ShenandoahBarrierSet::AccessKind kind() const;
virtual int Opcode() const;
virtual const Type* bottom_type() const;
virtual const Type* Value(PhaseGVN* phase) const;

6
src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp
View File

@ -61,12 +61,6 @@ bool ShenandoahAggressiveHeuristics::should_start_gc() const {
return true;
}
bool ShenandoahAggressiveHeuristics::should_process_references() {
if (!can_process_references()) return false;
// Randomly process refs with 50% chance.
return (os::random() & 1) == 1;
}
bool ShenandoahAggressiveHeuristics::should_unload_classes() {
if (!can_unload_classes_normal()) return false;
if (has_metaspace_oom()) return true;

2
src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp
View File

@ -37,8 +37,6 @@ public:
virtual bool should_start_gc() const;
virtual bool should_process_references();
virtual bool should_unload_classes();
virtual const char* name() { return "Aggressive"; }

12
src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp
View File

@ -257,18 +257,6 @@ void ShenandoahHeuristics::record_requested_gc() {
_gc_times_learned = 0;
}
bool ShenandoahHeuristics::can_process_references() {
if (ShenandoahRefProcFrequency == 0) return false;
return true;
}
bool ShenandoahHeuristics::should_process_references() {
if (!can_process_references()) return false;
size_t cycle = ShenandoahHeap::heap()->shenandoah_policy()->cycle_counter();
// Process references every Nth GC cycle.
return cycle % ShenandoahRefProcFrequency == 0;
}
bool ShenandoahHeuristics::can_unload_classes() {
if (!ClassUnloading) return false;
return true;

3
src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp
View File

@ -120,9 +120,6 @@ public:
virtual void choose_collection_set(ShenandoahCollectionSet* collection_set);
virtual bool can_process_references();
virtual bool should_process_references();
virtual bool can_unload_classes();
virtual bool can_unload_classes_normal();
virtual bool should_unload_classes();

5
src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp
View File

@ -36,11 +36,6 @@ bool ShenandoahPassiveHeuristics::should_start_gc() const {
return false;
}
bool ShenandoahPassiveHeuristics::should_process_references() {
// Always process references, if we can.
return can_process_references();
}
bool ShenandoahPassiveHeuristics::should_unload_classes() {
// Always unload classes, if we can.
return can_unload_classes();

2
src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp
View File

@ -31,8 +31,6 @@ class ShenandoahPassiveHeuristics : public ShenandoahHeuristics {
public:
virtual bool should_start_gc() const;
virtual bool should_process_references();
virtual bool should_unload_classes();
virtual bool should_degenerate_cycle();

21
src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp
View File

@ -68,7 +68,8 @@ void ShenandoahAsserts::print_obj(ShenandoahMessageBuffer& msg, oop obj) {
msg.append(" " PTR_FORMAT " - klass " PTR_FORMAT " %s\n", p2i(obj), p2i(obj->klass()), obj->klass()->external_name());
msg.append(" %3s allocated after mark start\n", ctx->allocated_after_mark_start(obj) ? "" : "not");
msg.append(" %3s after update watermark\n", cast_from_oop<HeapWord*>(obj) >= r->get_update_watermark() ? "" : "not");
msg.append(" %3s marked \n", ctx->is_marked(obj) ? "" : "not");
msg.append(" %3s marked strong\n", ctx->is_marked_strong(obj) ? "" : "not");
msg.append(" %3s marked weak\n", ctx->is_marked_weak(obj) ? "" : "not");
msg.append(" %3s in collection set\n", heap->in_collection_set(obj) ? "" : "not");
msg.append(" mark:%s\n", mw_ss.as_string());
msg.append(" region: %s", ss.as_string());
@ -353,24 +354,6 @@ void ShenandoahAsserts::print_rp_failure(const char *label, BoolObjectClosure* a
report_vm_error(file, line, msg.buffer());
}
void ShenandoahAsserts::assert_rp_isalive_not_installed(const char *file, int line) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ReferenceProcessor* rp = heap->ref_processor();
if (rp->is_alive_non_header() != NULL) {
print_rp_failure("Shenandoah assert_rp_isalive_not_installed failed", rp->is_alive_non_header(),
file, line);
}
}
void ShenandoahAsserts::assert_rp_isalive_installed(const char *file, int line) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ReferenceProcessor* rp = heap->ref_processor();
if (rp->is_alive_non_header() == NULL) {
print_rp_failure("Shenandoah assert_rp_isalive_installed failed", rp->is_alive_non_header(),
file, line);
}
}
void ShenandoahAsserts::assert_locked_or_shenandoah_safepoint(Mutex* lock, const char* file, int line) {
if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
return;

3
src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp
View File

@ -65,9 +65,6 @@ public:
static void assert_not_in_cset(void* interior_loc, oop obj, const char* file, int line);
static void assert_not_in_cset_loc(void* interior_loc, const char* file, int line);
static void assert_rp_isalive_not_installed(const char *file, int line);
static void assert_rp_isalive_installed(const char *file, int line);
static void assert_locked_or_shenandoah_safepoint(Mutex* lock, const char* file, int line);
static void assert_heaplocked(const char* file, int line);

21
src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
View File

@ -87,17 +87,6 @@ bool ShenandoahBarrierSet::need_load_reference_barrier(DecoratorSet decorators,
return is_reference_type(type);
}
bool ShenandoahBarrierSet::use_load_reference_barrier_weak(DecoratorSet decorators, BasicType type) {
assert(need_load_reference_barrier(decorators, type), "Should be subset of LRB");
assert(is_reference_type(type), "Why we here?");
// Native load reference barrier is only needed for concurrent root processing
if (!ShenandoahConcurrentRoots::can_do_concurrent_roots()) {
return false;
}
return ((decorators & IN_NATIVE) != 0) && ((decorators & ON_STRONG_OOP_REF) == 0);
}
bool ShenandoahBarrierSet::need_keep_alive_barrier(DecoratorSet decorators,BasicType type) {
if (!ShenandoahSATBBarrier) return false;
// Only needed for references
@ -109,6 +98,16 @@ bool ShenandoahBarrierSet::need_keep_alive_barrier(DecoratorSet decorators,Basic
return (on_weak_ref || unknown) && keep_alive;
}
ShenandoahBarrierSet::AccessKind ShenandoahBarrierSet::access_kind(DecoratorSet decorators, BasicType type) {
if ((decorators & IN_NATIVE) != 0) {
return AccessKind::NATIVE;
} else if ((decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF | ON_UNKNOWN_OOP_REF)) != 0) {
return AccessKind::WEAK;
} else {
return AccessKind::NORMAL;
}
}
void ShenandoahBarrierSet::on_thread_create(Thread* thread) {
// Create thread local data
ShenandoahThreadLocalData::create(thread);

15
src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp
View File

@ -33,8 +33,19 @@
class ShenandoahBarrierSetAssembler;
class ShenandoahBarrierSet: public BarrierSet {
private:
public:
enum class AccessKind {
// Regular in-heap access on reference fields
NORMAL,
// Off-heap reference access
NATIVE,
// In-heap reference access on referent fields of j.l.r.Reference objects
WEAK
};
private:
ShenandoahHeap* _heap;
BufferNode::Allocator _satb_mark_queue_buffer_allocator;
ShenandoahSATBMarkQueueSet _satb_mark_queue_set;
@ -53,8 +64,8 @@ public:
}
static bool need_load_reference_barrier(DecoratorSet decorators, BasicType type);
static bool use_load_reference_barrier_weak(DecoratorSet decorators, BasicType type);
static bool need_keep_alive_barrier(DecoratorSet decorators, BasicType type);
static AccessKind access_kind(DecoratorSet decorators, BasicType type);
void print_on(outputStream* st) const;

11
src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
View File

@ -198,7 +198,7 @@ template <typename T>
inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
oop value = Raw::oop_load_in_heap(addr);
ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set();
value = bs->load_reference_barrier(value);
value = bs->load_reference_barrier<decorators, T>(value, addr);
bs->keep_alive_if_weak<decorators>(value);
return value;
}
@ -207,9 +207,9 @@ template <DecoratorSet decorators, typename BarrierSetT>
inline oop ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
oop value = Raw::oop_load_in_heap_at(base, offset);
ShenandoahBarrierSet *const bs = ShenandoahBarrierSet::barrier_set();
value = bs->load_reference_barrier(value);
bs->keep_alive_if_weak(AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset),
value);
DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
value = bs->load_reference_barrier<decorators>(value, AccessInternal::oop_field_addr<decorators>(base, offset));
bs->keep_alive_if_weak(resolved_decorators, value);
return value;
}
@ -217,6 +217,7 @@ template <DecoratorSet decorators, typename BarrierSetT>
template <typename T>
inline void ShenandoahBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(T* addr, oop value) {
shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress());
shenandoah_assert_not_in_cset_if(addr, value, value != NULL && !ShenandoahHeap::heap()->cancelled_gc());
ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set();
bs->storeval_barrier(value);
bs->satb_barrier<decorators>(addr);
@ -339,7 +340,7 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o);
obj = fwd;
}
if (ENQUEUE && !ctx->is_marked(obj)) {
if (ENQUEUE && !ctx->is_marked_strong(obj)) {
queue.enqueue_known_active(obj);
}
}

406
src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp
View File

@ -31,8 +31,6 @@
#include "gc/shared/weakProcessor.inline.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTrace.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
@ -40,6 +38,7 @@
#include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
#include "gc/shenandoah/shenandoahMarkCompact.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
#include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
@ -61,7 +60,7 @@ private:
template <class T>
inline void do_oop_work(T* p) {
ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, NO_DEDUP>(p, _heap, _queue, _mark_context);
ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, NO_DEDUP>(p, _heap, _queue, _mark_context, false);
}
public:
@ -74,11 +73,12 @@ public:
void do_oop(oop* p) { do_oop_work(p); }
};
ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
MetadataVisitingOopIterateClosure(rp),
_queue(q),
_heap(ShenandoahHeap::heap()),
_mark_context(_heap->marking_context())
_mark_context(_heap->marking_context()),
_weak(false)
{ }
template<UpdateRefsMode UPDATE_REFS>
@ -153,14 +153,8 @@ public:
ShenandoahConcurrentWorkerSession worker_session(worker_id);
ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id);
ReferenceProcessor* rp;
if (heap->process_references()) {
rp = heap->ref_processor();
shenandoah_assert_rp_isalive_installed();
} else {
rp = NULL;
}
ShenandoahReferenceProcessor* rp = heap->ref_processor();
assert(rp != NULL, "need reference processor");
_cm->mark_loop(worker_id, _terminator, rp,
true, // cancellable
ShenandoahStringDedup::is_enabled()); // perform string dedup
@ -206,7 +200,7 @@ class ShenandoahProcessConcurrentRootsTask : public AbstractGangTask {
private:
ShenandoahConcurrentRootScanner<false /* concurrent */> _rs;
ShenandoahConcurrentMark* const _cm;
ReferenceProcessor* _rp;
ShenandoahReferenceProcessor* _rp;
public:
ShenandoahProcessConcurrentRootsTask(ShenandoahConcurrentMark* cm,
@ -222,12 +216,7 @@ ShenandoahProcessConcurrentRootsTask<T>::ShenandoahProcessConcurrentRootsTask(Sh
AbstractGangTask("Shenandoah Process Concurrent Roots"),
_rs(nworkers, phase),
_cm(cm),
_rp(NULL) {
ShenandoahHeap* heap = ShenandoahHeap::heap();
if (heap->process_references()) {
_rp = heap->ref_processor();
shenandoah_assert_rp_isalive_installed();
}
_rp(ShenandoahHeap::heap()->ref_processor()) {
}
template <typename T>
@ -253,13 +242,7 @@ public:
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahParallelWorkerSession worker_session(worker_id);
ReferenceProcessor* rp;
if (heap->process_references()) {
rp = heap->ref_processor();
shenandoah_assert_rp_isalive_installed();
} else {
rp = NULL;
}
ShenandoahReferenceProcessor* rp = heap->ref_processor();
// First drain remaining SATB buffers.
// Notice that this is not strictly necessary for mark-compact. But since
@ -303,9 +286,12 @@ void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_pha
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahGCPhase phase(root_phase);
ShenandoahReferenceProcessor* ref_processor = heap->ref_processor();
ref_processor->reset_thread_locals();
ref_processor->set_soft_reference_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
WorkGang* workers = heap->workers();
uint nworkers = workers->active_workers();
@ -407,21 +393,21 @@ void ShenandoahConcurrentMark::initialize(uint workers) {
// Mark concurrent roots during concurrent phases
class ShenandoahMarkConcurrentRootsTask : public AbstractGangTask {
private:
SuspendibleThreadSetJoiner _sts_joiner;
SuspendibleThreadSetJoiner _sts_joiner;
ShenandoahConcurrentRootScanner<true /* concurrent */> _rs;
ShenandoahObjToScanQueueSet* const _queue_set;
ReferenceProcessor* const _rp;
ShenandoahObjToScanQueueSet* const _queue_set;
ShenandoahReferenceProcessor* const _rp;
public:
ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
ReferenceProcessor* rp,
ShenandoahReferenceProcessor* rp,
ShenandoahPhaseTimings::Phase phase,
uint nworkers);
void work(uint worker_id);
};
ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs,
ReferenceProcessor* rp,
ShenandoahReferenceProcessor* rp,
ShenandoahPhaseTimings::Phase phase,
uint nworkers) :
AbstractGangTask("Shenandoah Concurrent Mark Roots"),
@ -442,19 +428,7 @@ void ShenandoahConcurrentMark::mark_from_roots() {
WorkGang* workers = _heap->workers();
uint nworkers = workers->active_workers();
ReferenceProcessor* rp = NULL;
if (_heap->process_references()) {
rp = _heap->ref_processor();
rp->set_active_mt_degree(nworkers);
// enable ("weak") refs discovery
rp->enable_discovery(true /*verify_no_refs*/);
rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
}
shenandoah_assert_rp_isalive_not_installed();
ShenandoahIsAliveSelector is_alive;
ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
ShenandoahReferenceProcessor* rp = _heap->ref_processor();
task_queues()->reserve(nworkers);
@ -480,10 +454,6 @@ void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) {
uint nworkers = _heap->workers()->active_workers();
{
shenandoah_assert_rp_isalive_not_installed();
ShenandoahIsAliveSelector is_alive;
ReferenceProcessorIsAliveMutator fix_isalive(_heap->ref_processor(), is_alive.is_alive_closure());
// Full GC does not execute concurrent cycle. Degenerated cycle may bypass concurrent cycle.
// In those cases, concurrent roots might not be scanned, scan them here. Ideally, this
// should piggyback to ShenandoahFinalMarkingTask, but it makes time tracking very hard.
@ -524,343 +494,11 @@ void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) {
assert(task_queues()->is_empty(), "Should be empty");
}
// When we're done marking everything, we process weak references.
if (_heap->process_references()) {
weak_refs_work(full_gc);
}
assert(task_queues()->is_empty(), "Should be empty");
TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats());
TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
}
// Weak Reference Closures
class ShenandoahCMDrainMarkingStackClosure: public VoidClosure {
uint _worker_id;
TaskTerminator* _terminator;
bool _reset_terminator;
public:
ShenandoahCMDrainMarkingStackClosure(uint worker_id, TaskTerminator* t, bool reset_terminator = false):
_worker_id(worker_id),
_terminator(t),
_reset_terminator(reset_terminator) {
}
void do_void() {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
ShenandoahHeap* sh = ShenandoahHeap::heap();
ShenandoahConcurrentMark* scm = sh->concurrent_mark();
assert(sh->process_references(), "why else would we be here?");
ReferenceProcessor* rp = sh->ref_processor();
shenandoah_assert_rp_isalive_installed();
scm->mark_loop(_worker_id, _terminator, rp,
false, // not cancellable
false); // do not do strdedup
if (_reset_terminator) {
_terminator->reset_for_reuse();
}
}
};
class ShenandoahCMKeepAliveClosure : public OopClosure {
private:
ShenandoahObjToScanQueue* _queue;
ShenandoahHeap* _heap;
ShenandoahMarkingContext* const _mark_context;
template <class T>
inline void do_oop_work(T* p) {
ShenandoahConcurrentMark::mark_through_ref<T, NONE, NO_DEDUP>(p, _heap, _queue, _mark_context);
}
public:
ShenandoahCMKeepAliveClosure(ShenandoahObjToScanQueue* q) :
_queue(q),
_heap(ShenandoahHeap::heap()),
_mark_context(_heap->marking_context()) {}
void do_oop(narrowOop* p) { do_oop_work(p); }
void do_oop(oop* p) { do_oop_work(p); }
};
class ShenandoahCMKeepAliveUpdateClosure : public OopClosure {
private:
ShenandoahObjToScanQueue* _queue;
ShenandoahHeap* _heap;
ShenandoahMarkingContext* const _mark_context;
template <class T>
inline void do_oop_work(T* p) {
ShenandoahConcurrentMark::mark_through_ref<T, SIMPLE, NO_DEDUP>(p, _heap, _queue, _mark_context);
}
public:
ShenandoahCMKeepAliveUpdateClosure(ShenandoahObjToScanQueue* q) :
_queue(q),
_heap(ShenandoahHeap::heap()),
_mark_context(_heap->marking_context()) {}
void do_oop(narrowOop* p) { do_oop_work(p); }
void do_oop(oop* p) { do_oop_work(p); }
};
class ShenandoahWeakUpdateClosure : public OopClosure {
private:
ShenandoahHeap* const _heap;
template <class T>
inline void do_oop_work(T* p) {
oop o = _heap->maybe_update_with_forwarded(p);
shenandoah_assert_marked_except(p, o, o == NULL);
}
public:
ShenandoahWeakUpdateClosure() : _heap(ShenandoahHeap::heap()) {}
void do_oop(narrowOop* p) { do_oop_work(p); }
void do_oop(oop* p) { do_oop_work(p); }
};
class ShenandoahRefProcTaskProxy : public AbstractGangTask {
private:
AbstractRefProcTaskExecutor::ProcessTask& _proc_task;
TaskTerminator* _terminator;
public:
ShenandoahRefProcTaskProxy(AbstractRefProcTaskExecutor::ProcessTask& proc_task,
TaskTerminator* t) :
AbstractGangTask("Shenandoah Process Weak References"),
_proc_task(proc_task),
_terminator(t) {
}
void work(uint worker_id) {
Thread* current_thread = Thread::current();
ResourceMark rm(current_thread);
HandleMark hm(current_thread);
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahCMDrainMarkingStackClosure complete_gc(worker_id, _terminator);
if (heap->has_forwarded_objects()) {
ShenandoahForwardedIsAliveClosure is_alive;
ShenandoahCMKeepAliveUpdateClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
_proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
} else {
ShenandoahIsAliveClosure is_alive;
ShenandoahCMKeepAliveClosure keep_alive(heap->concurrent_mark()->get_queue(worker_id));
_proc_task.work(worker_id, is_alive, keep_alive, complete_gc);
}
}
};
class ShenandoahRefProcTaskExecutor : public AbstractRefProcTaskExecutor {
private:
WorkGang* _workers;
public:
ShenandoahRefProcTaskExecutor(WorkGang* workers) :
_workers(workers) {
}
// Executes a task using worker threads.
void execute(ProcessTask& task, uint ergo_workers) {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahConcurrentMark* cm = heap->concurrent_mark();
ShenandoahPushWorkerQueuesScope scope(_workers, cm->task_queues(),
ergo_workers,
/* do_check = */ false);
uint nworkers = _workers->active_workers();
cm->task_queues()->reserve(nworkers);
TaskTerminator terminator(nworkers, cm->task_queues());
ShenandoahRefProcTaskProxy proc_task_proxy(task, &terminator);
_workers->run_task(&proc_task_proxy);
}
};
void ShenandoahConcurrentMark::weak_refs_work(bool full_gc) {
assert(_heap->process_references(), "sanity");
ShenandoahPhaseTimings::Phase phase_root =
full_gc ?
ShenandoahPhaseTimings::full_gc_weakrefs :
ShenandoahPhaseTimings::weakrefs;
ShenandoahGCPhase phase(phase_root);
ReferenceProcessor* rp = _heap->ref_processor();
// NOTE: We cannot shortcut on has_discovered_references() here, because
// we will miss marking JNI Weak refs then, see implementation in
// ReferenceProcessor::process_discovered_references.
weak_refs_work_doit(full_gc);
rp->verify_no_references_recorded();
assert(!rp->discovery_enabled(), "Post condition");
}
void ShenandoahConcurrentMark::weak_refs_work_doit(bool full_gc) {
ReferenceProcessor* rp = _heap->ref_processor();
ShenandoahPhaseTimings::Phase phase_process =
full_gc ?
ShenandoahPhaseTimings::full_gc_weakrefs_process :
ShenandoahPhaseTimings::weakrefs_process;
shenandoah_assert_rp_isalive_not_installed();
ShenandoahIsAliveSelector is_alive;
ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
WorkGang* workers = _heap->workers();
uint nworkers = workers->active_workers();
rp->setup_policy(_heap->soft_ref_policy()->should_clear_all_soft_refs());
rp->set_active_mt_degree(nworkers);
assert(task_queues()->is_empty(), "Should be empty");
// complete_gc and keep_alive closures instantiated here are only needed for
// single-threaded path in RP. They share the queue 0 for tracking work, which
// simplifies implementation. Since RP may decide to call complete_gc several
// times, we need to be able to reuse the terminator.
uint serial_worker_id = 0;
TaskTerminator terminator(1, task_queues());
ShenandoahCMDrainMarkingStackClosure complete_gc(serial_worker_id, &terminator, /* reset_terminator = */ true);
ShenandoahRefProcTaskExecutor executor(workers);
ReferenceProcessorPhaseTimes pt(_heap->gc_timer(), rp->num_queues());
{
// Note: Don't emit JFR event for this phase, to avoid overflow nesting phase level.
// Reference Processor emits 2 levels JFR event, that can get us over the JFR
// event nesting level limits, in case of degenerated GC gets upgraded to
// full GC.
ShenandoahTimingsTracker phase_timing(phase_process);
if (_heap->has_forwarded_objects()) {
ShenandoahCMKeepAliveUpdateClosure keep_alive(get_queue(serial_worker_id));
const ReferenceProcessorStats& stats =
rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
&complete_gc, &executor,
&pt);
_heap->tracer()->report_gc_reference_stats(stats);
} else {
ShenandoahCMKeepAliveClosure keep_alive(get_queue(serial_worker_id));
const ReferenceProcessorStats& stats =
rp->process_discovered_references(is_alive.is_alive_closure(), &keep_alive,
&complete_gc, &executor,
&pt);
_heap->tracer()->report_gc_reference_stats(stats);
}
pt.print_all_references();
assert(task_queues()->is_empty(), "Should be empty");
}
}
class ShenandoahCancelledGCYieldClosure : public YieldClosure {
private:
ShenandoahHeap* const _heap;
public:
ShenandoahCancelledGCYieldClosure() : _heap(ShenandoahHeap::heap()) {};
virtual bool should_return() { return _heap->cancelled_gc(); }
};
class ShenandoahPrecleanCompleteGCClosure : public VoidClosure {
public:
void do_void() {
ShenandoahHeap* sh = ShenandoahHeap::heap();
ShenandoahConcurrentMark* scm = sh->concurrent_mark();
assert(sh->process_references(), "why else would we be here?");
TaskTerminator terminator(1, scm->task_queues());
ReferenceProcessor* rp = sh->ref_processor();
shenandoah_assert_rp_isalive_installed();
scm->mark_loop(0, &terminator, rp,
false, // not cancellable
false); // do not do strdedup
}
};
class ShenandoahPrecleanTask : public AbstractGangTask {
private:
ReferenceProcessor* _rp;
public:
ShenandoahPrecleanTask(ReferenceProcessor* rp) :
AbstractGangTask("Shenandoah Precleaning"),
_rp(rp) {}
void work(uint worker_id) {
assert(worker_id == 0, "The code below is single-threaded, only one worker is expected");
ShenandoahParallelWorkerSession worker_session(worker_id);
ShenandoahHeap* sh = ShenandoahHeap::heap();
assert(!sh->has_forwarded_objects(), "No forwarded objects expected here");
ShenandoahObjToScanQueue* q = sh->concurrent_mark()->get_queue(worker_id);
ShenandoahCancelledGCYieldClosure yield;
ShenandoahPrecleanCompleteGCClosure complete_gc;
ShenandoahIsAliveClosure is_alive;
ShenandoahCMKeepAliveClosure keep_alive(q);
ResourceMark rm;
_rp->preclean_discovered_references(&is_alive, &keep_alive,
&complete_gc, &yield,
NULL);
}
};
void ShenandoahConcurrentMark::preclean_weak_refs() {
// Pre-cleaning weak references before diving into STW makes sense at the
// end of concurrent mark. This will filter out the references which referents
// are alive. Note that ReferenceProcessor already filters out these on reference
// discovery, and the bulk of work is done here. This phase processes leftovers
// that missed the initial filtering, i.e. when referent was marked alive after
// reference was discovered by RP.
assert(_heap->process_references(), "sanity");
// Shortcut if no references were discovered to avoid winding up threads.
ReferenceProcessor* rp = _heap->ref_processor();
if (!rp->has_discovered_references()) {
return;
}
assert(task_queues()->is_empty(), "Should be empty");
ReferenceProcessorMTDiscoveryMutator fix_mt_discovery(rp, false);
shenandoah_assert_rp_isalive_not_installed();
ShenandoahIsAliveSelector is_alive;
ReferenceProcessorIsAliveMutator fix_isalive(rp, is_alive.is_alive_closure());
// Execute precleaning in the worker thread: it will give us GCLABs, String dedup
// queues and other goodies. When upstream ReferenceProcessor starts supporting
// parallel precleans, we can extend this to more threads.
WorkGang* workers = _heap->workers();
uint nworkers = workers->active_workers();
assert(nworkers == 1, "This code uses only a single worker");
task_queues()->reserve(nworkers);
ShenandoahPrecleanTask task(rp);
workers->run_task(&task);
assert(task_queues()->is_empty(), "Should be empty");
}
void ShenandoahConcurrentMark::cancel() {
// Clean up marking stacks.
ShenandoahObjToScanQueueSet* queues = task_queues();
@ -876,7 +514,7 @@ ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) {
}
template <bool CANCELLABLE>
void ShenandoahConcurrentMark::mark_loop_prework(uint w, TaskTerminator *t, ReferenceProcessor *rp,
void ShenandoahConcurrentMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor* rp,
bool strdedup) {
ShenandoahObjToScanQueue* q = get_queue(w);
@ -934,6 +572,8 @@ void ShenandoahConcurrentMark::mark_loop_work(T* cl, ShenandoahLiveData* live_da
ShenandoahObjToScanQueue* q;
ShenandoahMarkTask t;
_heap->ref_processor()->set_mark_closure(worker_id, cl);
/*
* Process outstanding queues, if any.
*

20
src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp
View File

@ -32,6 +32,7 @@
#include "gc/shenandoah/shenandoahTaskqueue.hpp"
class ShenandoahStrDedupQueue;
class ShenandoahReferenceProcessor;
class ShenandoahConcurrentMark: public CHeapObj<mtGC> {
private:
@ -49,10 +50,10 @@ private:
inline void do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, ShenandoahMarkTask* task);
template <class T>
inline void do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop array);
inline void do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop array, bool weak);
template <class T>
inline void do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop array, int chunk, int pow);
inline void do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop array, int chunk, int pow, bool weak);
inline void count_liveness(ShenandoahLiveData* live_data, oop obj);
@ -60,10 +61,10 @@ private:
void mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *t);
template <bool CANCELLABLE>
void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ReferenceProcessor *rp, bool strdedup);
void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor* rp, bool strdedup);
public:
void mark_loop(uint worker_id, TaskTerminator* terminator, ReferenceProcessor *rp,
void mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor* rp,
bool cancellable, bool strdedup) {
if (cancellable) {
mark_loop_prework<true>(worker_id, terminator, rp, strdedup);
@ -73,7 +74,7 @@ public:
}
template<class T, UpdateRefsMode UPDATE_REFS, StringDedupMode STRING_DEDUP>
static inline void mark_through_ref(T* p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context);
static inline void mark_through_ref(T* p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak);
void mark_from_roots();
void finish_mark_from_roots(bool full_gc);
@ -82,15 +83,6 @@ public:
void update_roots(ShenandoahPhaseTimings::Phase root_phase);
void update_thread_roots(ShenandoahPhaseTimings::Phase root_phase);
// ---------- Weak references
//
private:
void weak_refs_work(bool full_gc);
void weak_refs_work_doit(bool full_gc);
public:
void preclean_weak_refs();
// ---------- Helpers
// Used from closures, need to be public
//

41
src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved.
* Copyright (c) 2015, 2020, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -45,6 +45,10 @@ void ShenandoahConcurrentMark::do_task(ShenandoahObjToScanQueue* q, T* cl, Shena
shenandoah_assert_marked(NULL, obj);
shenandoah_assert_not_in_cset_except(NULL, obj, _heap->cancelled_gc());
// Are we in weak subgraph scan?
bool weak = task->is_weak();
cl->set_weak(weak);
if (task->is_not_chunked()) {
if (obj->is_instance()) {
// Case 1: Normal oop, process as usual.
@ -52,7 +56,7 @@ void ShenandoahConcurrentMark::do_task(ShenandoahObjToScanQueue* q, T* cl, Shena
} else if (obj->is_objArray()) {
// Case 2: Object array instance and no chunk is set. Must be the first
// time we visit it, start the chunked processing.
do_chunked_array_start<T>(q, cl, obj);
do_chunked_array_start<T>(q, cl, obj, weak);
} else {
// Case 3: Primitive array. Do nothing, no oops there. We use the same
// performance tweak TypeArrayKlass::oop_oop_iterate_impl is using:
@ -61,10 +65,14 @@ void ShenandoahConcurrentMark::do_task(ShenandoahObjToScanQueue* q, T* cl, Shena
assert (obj->is_typeArray(), "should be type array");
}
// Count liveness the last: push the outstanding work to the queues first
count_liveness(live_data, obj);
// Avoid double-counting objects that are visited twice due to upgrade
// from final- to strong mark.
if (task->count_liveness()) {
count_liveness(live_data, obj);
}
} else {
// Case 4: Array chunk, has sensible chunk id. Process it.
do_chunked_array<T>(q, cl, obj, task->chunk(), task->pow());
do_chunked_array<T>(q, cl, obj, task->chunk(), task->pow(), weak);
}
}
@ -98,7 +106,7 @@ inline void ShenandoahConcurrentMark::count_liveness(ShenandoahLiveData* live_da
}
template <class T>
inline void ShenandoahConcurrentMark::do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop obj) {
inline void ShenandoahConcurrentMark::do_chunked_array_start(ShenandoahObjToScanQueue* q, T* cl, oop obj, bool weak) {
assert(obj->is_objArray(), "expect object array");
objArrayOop array = objArrayOop(obj);
int len = array->length();
@ -129,7 +137,7 @@ inline void ShenandoahConcurrentMark::do_chunked_array_start(ShenandoahObjToScan
pow--;
chunk = 2;
last_idx = (1 << pow);
bool pushed = q->push(ShenandoahMarkTask(array, 1, pow));
bool pushed = q->push(ShenandoahMarkTask(array, true, weak, 1, pow));
assert(pushed, "overflow queue should always succeed pushing");
}
@ -142,7 +150,7 @@ inline void ShenandoahConcurrentMark::do_chunked_array_start(ShenandoahObjToScan
int right_chunk = chunk*2;
int left_chunk_end = left_chunk * (1 << pow);
if (left_chunk_end < len) {
bool pushed = q->push(ShenandoahMarkTask(array, left_chunk, pow));
bool pushed = q->push(ShenandoahMarkTask(array, true, weak, left_chunk, pow));
assert(pushed, "overflow queue should always succeed pushing");
chunk = right_chunk;
last_idx = left_chunk_end;
@ -160,7 +168,7 @@ inline void ShenandoahConcurrentMark::do_chunked_array_start(ShenandoahObjToScan
}
template <class T>
inline void ShenandoahConcurrentMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop obj, int chunk, int pow) {
inline void ShenandoahConcurrentMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, oop obj, int chunk, int pow, bool weak) {
assert(obj->is_objArray(), "expect object array");
objArrayOop array = objArrayOop(obj);
@ -171,7 +179,7 @@ inline void ShenandoahConcurrentMark::do_chunked_array(ShenandoahObjToScanQueue*
while ((1 << pow) > (int)ObjArrayMarkingStride && (chunk*2 < ShenandoahMarkTask::chunk_size())) {
pow--;
chunk *= 2;
bool pushed = q->push(ShenandoahMarkTask(array, chunk - 1, pow));
bool pushed = q->push(ShenandoahMarkTask(array, true, weak, chunk - 1, pow));
assert(pushed, "overflow queue should always succeed pushing");
}
@ -215,13 +223,13 @@ public:
void do_buffer_impl(void **buffer, size_t size) {
for (size_t i = 0; i < size; ++i) {
oop *p = (oop *) &buffer[i];
ShenandoahConcurrentMark::mark_through_ref<oop, NONE, STRING_DEDUP>(p, _heap, _queue, _mark_context);
ShenandoahConcurrentMark::mark_through_ref<oop, NONE, STRING_DEDUP>(p, _heap, _queue, _mark_context, false);
}
}
};
template<class T, UpdateRefsMode UPDATE_REFS, StringDedupMode STRING_DEDUP>
inline void ShenandoahConcurrentMark::mark_through_ref(T *p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context) {
inline void ShenandoahConcurrentMark::mark_through_ref(T *p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak) {
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
@ -252,8 +260,15 @@ inline void ShenandoahConcurrentMark::mark_through_ref(T *p, ShenandoahHeap* hea
shenandoah_assert_not_forwarded(p, obj);
shenandoah_assert_not_in_cset_except(p, obj, heap->cancelled_gc());
if (mark_context->mark(obj)) {
bool pushed = q->push(ShenandoahMarkTask(obj));
bool skip_live = false;
bool marked;
if (weak) {
marked = mark_context->mark_weak(obj);
} else {
marked = mark_context->mark_strong(obj, /* was_upgraded = */ skip_live);
}
if (marked) {
bool pushed = q->push(ShenandoahMarkTask(obj, skip_live, weak));
assert(pushed, "overflow queue should always succeed pushing");
if ((STRING_DEDUP == ENQUEUE_DEDUP) && ShenandoahStringDedup::is_candidate(obj)) {

7
src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
View File

@ -141,7 +141,6 @@ void ShenandoahControlThread::run_service() {
policy->record_explicit_to_concurrent();
mode = default_mode;
// Unload and clean up everything
heap->set_process_references(heuristics->can_process_references());
heap->set_unload_classes(heuristics->can_unload_classes());
} else {
policy->record_explicit_to_full();
@ -158,7 +157,6 @@ void ShenandoahControlThread::run_service() {
mode = default_mode;
// Unload and clean up everything
heap->set_process_references(heuristics->can_process_references());
heap->set_unload_classes(heuristics->can_unload_classes());
} else {
policy->record_implicit_to_full();
@ -172,7 +170,6 @@ void ShenandoahControlThread::run_service() {
}
// Ask policy if this cycle wants to process references or unload classes
heap->set_process_references(heuristics->should_process_references());
heap->set_unload_classes(heuristics->should_unload_classes());
}
@ -404,14 +401,12 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cau
heap->entry_mark();
if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_mark)) return;
// If not cancelled, can try to concurrently pre-clean
heap->entry_preclean();
// Complete marking under STW, and start evacuation
heap->vmop_entry_final_mark();
// Process weak roots that might still point to regions that would be broken by cleanup
if (heap->is_concurrent_weak_root_in_progress()) {
heap->entry_weak_refs();
heap->entry_weak_roots();
}

119
src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
View File

@ -55,6 +55,7 @@
#include "gc/shenandoah/shenandoahPacer.inline.hpp"
#include "gc/shenandoah/shenandoahPadding.hpp"
#include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
#include "gc/shenandoah/shenandoahStringDedup.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.hpp"
@ -206,10 +207,10 @@ jint ShenandoahHeap::initialize() {
// Reserve and commit memory for bitmap(s)
//
_bitmap_size = MarkBitMap::compute_size(heap_rs.size());
_bitmap_size = ShenandoahMarkBitMap::compute_size(heap_rs.size());
_bitmap_size = align_up(_bitmap_size, bitmap_page_size);
size_t bitmap_bytes_per_region = reg_size_bytes / MarkBitMap::heap_map_factor();
size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
guarantee(bitmap_bytes_per_region != 0,
"Bitmap bytes per region should not be zero");
@ -393,9 +394,6 @@ jint ShenandoahHeap::initialize() {
_control_thread = new ShenandoahControlThread();
_ref_proc_mt_processing = ParallelRefProcEnabled && (ParallelGCThreads > 1);
_ref_proc_mt_discovery = _max_workers > 1;
ShenandoahInitLogger::print();
return JNI_OK;
@ -475,7 +473,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
_gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()),
_soft_ref_policy(),
_log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
_ref_processor(NULL),
_ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))),
_marking_context(NULL),
_bitmap_size(0),
_bitmap_regions_per_slice(0),
@ -615,8 +613,6 @@ void ShenandoahHeap::post_initialize() {
_scm->initialize(_max_workers);
_full_gc->initialize(_gc_timer);
ref_processing_init();
_heuristics->initialize();
JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
@ -1791,13 +1787,9 @@ void ShenandoahHeap::op_final_mark() {
concurrent_mark()->cancel();
set_concurrent_mark_in_progress(false);
if (process_references()) {
// Abandon reference processing right away: pre-cleaning must have failed.
ReferenceProcessor *rp = ref_processor();
rp->disable_discovery();
rp->abandon_partial_discovery();
rp->verify_no_references_recorded();
}
// Abandon reference processing right away: pre-cleaning must have failed.
ShenandoahReferenceProcessor* rp = ref_processor();
rp->abandon_partial_discovery();
}
}
@ -2004,6 +1996,15 @@ public:
}
};
void ShenandoahHeap::op_weak_refs() {
// Concurrent weak refs processing
{
ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_refs_work);
ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs_work);
ref_processor()->process_references(workers(), true /* concurrent */);
}
}
void ShenandoahHeap::op_weak_roots() {
if (is_concurrent_weak_root_in_progress()) {
// Concurrent weak root processing
@ -2077,13 +2078,6 @@ void ShenandoahHeap::op_reset() {
parallel_heap_region_iterate(&cl);
}
void ShenandoahHeap::op_preclean() {
if (ShenandoahPacing) {
pacer()->setup_for_preclean();
}
concurrent_mark()->preclean_weak_refs();
}
void ShenandoahHeap::op_full(GCCause::Cause cause) {
ShenandoahMetricsSnapshot metrics;
metrics.snap_before();
@ -2125,7 +2119,6 @@ void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
//
// Note that we can only do this for "outside-cycle" degens, otherwise we would risk
// changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
set_process_references(heuristics()->can_process_references());
set_unload_classes(heuristics()->can_unload_classes());
op_reset();
@ -2150,6 +2143,12 @@ void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) {
ShenandoahCodeRoots::disarm_nmethods();
}
{
ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_refs_work);
ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs_work);
ref_processor()->process_references(workers(), false /* concurrent */);
}
op_cleanup_early();
case _degenerated_evac:
@ -2310,22 +2309,6 @@ void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool in_progress) {
}
}
void ShenandoahHeap::ref_processing_init() {
assert(_max_workers > 0, "Sanity");
_ref_processor =
new ReferenceProcessor(&_subject_to_discovery, // is_subject_to_discovery
_ref_proc_mt_processing, // MT processing
_max_workers, // Degree of MT processing
_ref_proc_mt_discovery, // MT discovery
_max_workers, // Degree of MT discovery
false, // Reference discovery is not atomic
NULL, // No closure, should be installed before use
true); // Scale worker threads
shenandoah_assert_rp_isalive_not_installed();
}
GCTracer* ShenandoahHeap::tracer() {
return shenandoah_policy()->tracer();
}
@ -2461,18 +2444,10 @@ void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
set_gc_state_mask(HAS_FORWARDED, cond);
}
void ShenandoahHeap::set_process_references(bool pr) {
_process_references.set_cond(pr);
}
void ShenandoahHeap::set_unload_classes(bool uc) {
_unload_classes.set_cond(uc);
}
bool ShenandoahHeap::process_references() const {
return _process_references.is_set();
}
bool ShenandoahHeap::unload_classes() const {
return _unload_classes.is_set();
}
@ -3067,6 +3042,19 @@ void ShenandoahHeap::entry_updaterefs() {
op_updaterefs();
}
void ShenandoahHeap::entry_weak_refs() {
static const char* msg = "Concurrent weak references";
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
EventMark em("%s", msg);
ShenandoahWorkerScope scope(workers(),
ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
"concurrent weak references");
try_inject_alloc_failure();
op_weak_refs();
}
void ShenandoahHeap::entry_weak_roots() {
static const char* msg = "Concurrent weak roots";
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
@ -3153,22 +3141,6 @@ void ShenandoahHeap::entry_reset() {
op_reset();
}
void ShenandoahHeap::entry_preclean() {
if (ShenandoahPreclean && process_references()) {
static const char* msg = "Concurrent precleaning";
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_preclean);
EventMark em("%s", msg);
ShenandoahWorkerScope scope(workers(),
ShenandoahWorkerPolicy::calc_workers_for_conc_preclean(),
"concurrent preclean",
/* check_workers = */ false);
try_inject_alloc_failure();
op_preclean();
}
}
void ShenandoahHeap::entry_uncommit(double shrink_before, size_t shrink_until) {
static const char *msg = "Concurrent uncommit";
ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_uncommit, true /* log_heap_usage */);
@ -3245,14 +3217,9 @@ void ShenandoahHeap::deduplicate_string(oop str) {
const char* ShenandoahHeap::init_mark_event_message() const {
assert(!has_forwarded_objects(), "Should not have forwarded objects here");
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (proc_refs && unload_cls) {
return "Pause Init Mark (process weakrefs) (unload classes)";
} else if (proc_refs) {
return "Pause Init Mark (process weakrefs)";
} else if (unload_cls) {
if (unload_cls) {
return "Pause Init Mark (unload classes)";
} else {
return "Pause Init Mark";
@ -3262,14 +3229,9 @@ const char* ShenandoahHeap::init_mark_event_message() const {
const char* ShenandoahHeap::final_mark_event_message() const {
assert(!has_forwarded_objects(), "Should not have forwarded objects here");
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (proc_refs && unload_cls) {
return "Pause Final Mark (process weakrefs) (unload classes)";
} else if (proc_refs) {
return "Pause Final Mark (process weakrefs)";
} else if (unload_cls) {
if (unload_cls) {
return "Pause Final Mark (unload classes)";
} else {
return "Pause Final Mark";
@ -3279,14 +3241,9 @@ const char* ShenandoahHeap::final_mark_event_message() const {
const char* ShenandoahHeap::conc_mark_event_message() const {
assert(!has_forwarded_objects(), "Should not have forwarded objects here");
bool proc_refs = process_references();
bool unload_cls = unload_classes();
if (proc_refs && unload_cls) {
return "Concurrent marking (process weakrefs) (unload classes)";
} else if (proc_refs) {
return "Concurrent marking (process weakrefs)";
} else if (unload_cls) {
if (unload_cls) {
return "Concurrent marking (unload classes)";
} else {
return "Concurrent marking";

21
src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
View File

@ -42,7 +42,6 @@
class ConcurrentGCTimer;
class ObjectIterateScanRootClosure;
class ReferenceProcessor;
class ShenandoahCollectorPolicy;
class ShenandoahControlThread;
class ShenandoahGCSession;
@ -60,7 +59,7 @@ class ShenandoahFreeSet;
class ShenandoahConcurrentMark;
class ShenandoahMarkCompact;
class ShenandoahMonitoringSupport;
class ShenandoahObjToScanQueueSet;
class ShenandoahReferenceProcessor;
class ShenandoahPacer;
class ShenandoahVerifier;
class ShenandoahWorkGang;
@ -390,7 +389,7 @@ public:
// for concurrent operation.
void entry_reset();
void entry_mark();
void entry_preclean();
void entry_weak_refs();
void entry_weak_roots();
void entry_class_unloading();
void entry_strong_roots();
@ -415,7 +414,7 @@ private:
void op_reset();
void op_mark();
void op_preclean();
void op_weak_refs();
void op_weak_roots();
void op_class_unloading();
void op_strong_roots();
@ -494,20 +493,10 @@ public:
// ---------- Reference processing
//
private:
AlwaysTrueClosure _subject_to_discovery;
ReferenceProcessor* _ref_processor;
ShenandoahSharedFlag _process_references;
bool _ref_proc_mt_discovery;
bool _ref_proc_mt_processing;
void ref_processing_init();
ShenandoahReferenceProcessor* const _ref_processor;
public:
ReferenceProcessor* ref_processor() { return _ref_processor; }
bool ref_processor_mt_discovery() { return _ref_proc_mt_discovery; }
bool ref_processor_mt_processing() { return _ref_proc_mt_processing; }
void set_process_references(bool pr);
bool process_references() const;
ShenandoahReferenceProcessor* ref_processor() { return _ref_processor; }
// ---------- Class Unloading
//

7
src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
View File

@ -401,7 +401,6 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
ShenandoahMarkingContext* const ctx = complete_marking_context();
assert(ctx->is_complete(), "sanity");
MarkBitMap* mark_bit_map = ctx->mark_bit_map();
HeapWord* tams = ctx->top_at_mark_start(region);
size_t skip_bitmap_delta = 1;
@ -413,7 +412,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
// Try to scan the initial candidate. If the candidate is above the TAMS, it would
// fail the subsequent "< limit_bitmap" checks, and fall through to Step 2.
HeapWord* cb = mark_bit_map->get_next_marked_addr(start, end);
HeapWord* cb = ctx->get_next_marked_addr(start, end);
intx dist = ShenandoahMarkScanPrefetch;
if (dist > 0) {
@ -440,7 +439,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
slots[avail++] = cb;
cb += skip_bitmap_delta;
if (cb < limit_bitmap) {
cb = mark_bit_map->get_next_marked_addr(cb, limit_bitmap);
cb = ctx->get_next_marked_addr(cb, limit_bitmap);
}
}
@ -463,7 +462,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
cl->do_object(obj);
cb += skip_bitmap_delta;
if (cb < limit_bitmap) {
cb = mark_bit_map->get_next_marked_addr(cb, limit_bitmap);
cb = ctx->get_next_marked_addr(cb, limit_bitmap);
}
}
}

4
src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp
View File

@ -57,10 +57,6 @@ void ShenandoahInitLogger::print_heap() {
log_info(gc, init)("Humongous Object Threshold: " SIZE_FORMAT "%s",
byte_size_in_exact_unit(ShenandoahHeapRegion::humongous_threshold_bytes()),
exact_unit_for_byte_size(ShenandoahHeapRegion::humongous_threshold_bytes()));
log_info(gc, init)("Reference Processing: %s discovery, %s processing",
heap->ref_processor_mt_discovery() ? "Parallel" : "Serial",
heap->ref_processor_mt_processing() ? "Parallel" : "Serial");
}
void ShenandoahInitLogger::print() {

146
src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp Normal file
View File

@ -0,0 +1,146 @@
/*
* Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Red Hat, Inc. and/or its affiliates.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahMarkBitMap.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "utilities/globalDefinitions.hpp"
ShenandoahMarkBitMap::ShenandoahMarkBitMap(MemRegion heap, MemRegion storage) :
_shift(LogMinObjAlignment),
_covered(heap),
_map((BitMap::bm_word_t*) storage.start()),
_size((heap.word_size() * 2) >> _shift) {
}
size_t ShenandoahMarkBitMap::compute_size(size_t heap_size) {
return ReservedSpace::allocation_align_size_up(heap_size / mark_distance());
}
size_t ShenandoahMarkBitMap::mark_distance() {
return MinObjAlignmentInBytes * BitsPerByte / 2;
}
HeapWord* ShenandoahMarkBitMap::get_next_marked_addr(const HeapWord* addr,
const HeapWord* limit) const {
assert(limit != NULL, "limit must not be NULL");
// Round addr up to a possible object boundary to be safe.
size_t const addr_offset = address_to_index(align_up(addr, HeapWordSize << LogMinObjAlignment));
size_t const limit_offset = address_to_index(limit);
size_t const nextOffset = get_next_one_offset(addr_offset, limit_offset);
return index_to_address(nextOffset);
}
void ShenandoahMarkBitMap::clear_range_within_word(idx_t beg, idx_t end) {
// With a valid range (beg <= end), this test ensures that end != 0, as
// required by inverted_bit_mask_for_range. Also avoids an unnecessary write.
if (beg != end) {
bm_word_t mask = inverted_bit_mask_for_range(beg, end);
*word_addr(beg) &= mask;
}
}
void ShenandoahMarkBitMap::clear_range(idx_t beg, idx_t end) {
verify_range(beg, end);
idx_t beg_full_word = to_words_align_up(beg);
idx_t end_full_word = to_words_align_down(end);
if (beg_full_word < end_full_word) {
// The range includes at least one full word.
clear_range_within_word(beg, bit_index(beg_full_word));
clear_range_of_words(beg_full_word, end_full_word);
clear_range_within_word(bit_index(end_full_word), end);
} else {
// The range spans at most 2 partial words.
idx_t boundary = MIN2(bit_index(beg_full_word), end);
clear_range_within_word(beg, boundary);
clear_range_within_word(boundary, end);
}
}
bool ShenandoahMarkBitMap::is_small_range_of_words(idx_t beg_full_word, idx_t end_full_word) {
// There is little point to call large version on small ranges.
// Need to check carefully, keeping potential idx_t over/underflow in mind,
// because beg_full_word > end_full_word can occur when beg and end are in
// the same word.
// The threshold should be at least one word.
STATIC_ASSERT(small_range_words >= 1);
return beg_full_word + small_range_words >= end_full_word;
}
void ShenandoahMarkBitMap::clear_large_range(idx_t beg, idx_t end) {
verify_range(beg, end);
idx_t beg_full_word = to_words_align_up(beg);
idx_t end_full_word = to_words_align_down(end);
if (is_small_range_of_words(beg_full_word, end_full_word)) {
clear_range(beg, end);
return;
}
// The range includes at least one full word.
clear_range_within_word(beg, bit_index(beg_full_word));
clear_large_range_of_words(beg_full_word, end_full_word);
clear_range_within_word(bit_index(end_full_word), end);
}
void ShenandoahMarkBitMap::clear_range_large(MemRegion mr) {
MemRegion intersection = mr.intersection(_covered);
assert(!intersection.is_empty(),
"Given range from " PTR_FORMAT " to " PTR_FORMAT " is completely outside the heap",
p2i(mr.start()), p2i(mr.end()));
// convert address range into offset range
size_t beg = address_to_index(intersection.start());
size_t end = address_to_index(intersection.end());
clear_large_range(beg, end);
}
#ifdef ASSERT
void ShenandoahMarkBitMap::check_mark(HeapWord* addr) const {
assert(ShenandoahHeap::heap()->is_in(addr),
"Trying to access bitmap " PTR_FORMAT " for address " PTR_FORMAT " not in the heap.",
p2i(this), p2i(addr));
}
void ShenandoahMarkBitMap::verify_index(idx_t bit) const {
assert(bit < _size,
"BitMap index out of bounds: " SIZE_FORMAT " >= " SIZE_FORMAT,
bit, _size);
}
void ShenandoahMarkBitMap::verify_limit(idx_t bit) const {
assert(bit <= _size,
"BitMap limit out of bounds: " SIZE_FORMAT " > " SIZE_FORMAT,
bit, _size);
}
void ShenandoahMarkBitMap::verify_range(idx_t beg, idx_t end) const {
assert(beg <= end,
"BitMap range error: " SIZE_FORMAT " > " SIZE_FORMAT, beg, end);
verify_limit(end);
}
#endif

180
src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp Normal file
View File

@ -0,0 +1,180 @@
/*
* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Red Hat, Inc. and/or its affiliates.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKBITMAP_HPP
#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKBITMAP_HPP
#include "memory/memRegion.hpp"
#include "runtime/atomic.hpp"
#include "utilities/globalDefinitions.hpp"
class ShenandoahMarkBitMap {
public:
typedef size_t idx_t; // Type used for bit and word indices.
typedef uintptr_t bm_word_t; // Element type of array that represents the
// bitmap, with BitsPerWord bits per element.
private:
// Values for get_next_bit_impl flip parameter.
static const bm_word_t find_ones_flip = 0;
static const bm_word_t find_zeros_flip = ~(bm_word_t)0;
int const _shift;
MemRegion _covered;
bm_word_t* _map; // First word in bitmap
idx_t _size; // Size of bitmap (in bits)
// Threshold for performing small range operation, even when large range
// operation was requested. Measured in words.
static const size_t small_range_words = 32;
static bool is_small_range_of_words(idx_t beg_full_word, idx_t end_full_word);
inline size_t address_to_index(const HeapWord* addr) const;
inline HeapWord* index_to_address(size_t offset) const;
void check_mark(HeapWord* addr) const NOT_DEBUG_RETURN;
// Return a mask that will select the specified bit, when applied to the word
// containing the bit.
static bm_word_t bit_mask(idx_t bit) { return (bm_word_t)1 << bit_in_word(bit); }
// Return the bit number of the first bit in the specified word.
static idx_t bit_index(idx_t word) { return word << LogBitsPerWord; }
// Return the position of bit within the word that contains it (e.g., if
// bitmap words are 32 bits, return a number 0 <= n <= 31).
static idx_t bit_in_word(idx_t bit) { return bit & (BitsPerWord - 1); }
bm_word_t* map() { return _map; }
const bm_word_t* map() const { return _map; }
bm_word_t map(idx_t word) const { return _map[word]; }
// Return a pointer to the word containing the specified bit.
bm_word_t* word_addr(idx_t bit) {
return map() + to_words_align_down(bit);
}
const bm_word_t* word_addr(idx_t bit) const {
return map() + to_words_align_down(bit);
}
static inline const bm_word_t load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order);
bool at(idx_t index) const {
verify_index(index);
return (*word_addr(index) & bit_mask(index)) != 0;
}
// Assumes relevant validity checking for bit has already been done.
static idx_t raw_to_words_align_up(idx_t bit) {
return raw_to_words_align_down(bit + (BitsPerWord - 1));
}
// Assumes relevant validity checking for bit has already been done.
static idx_t raw_to_words_align_down(idx_t bit) {
return bit >> LogBitsPerWord;
}
// Word-aligns bit and converts it to a word offset.
// precondition: bit <= size()
idx_t to_words_align_up(idx_t bit) const {
verify_limit(bit);
return raw_to_words_align_up(bit);
}
// Word-aligns bit and converts it to a word offset.
// precondition: bit <= size()
inline idx_t to_words_align_down(idx_t bit) const {
verify_limit(bit);
return raw_to_words_align_down(bit);
}
// Helper for get_next_{zero,one}_bit variants.
// - flip designates whether searching for 1s or 0s. Must be one of
// find_{zeros,ones}_flip.
// - aligned_right is true if r_index is a priori on a bm_word_t boundary.
template<bm_word_t flip, bool aligned_right>
inline idx_t get_next_bit_impl(idx_t l_index, idx_t r_index) const;
inline idx_t get_next_one_offset (idx_t l_index, idx_t r_index) const;
void clear_large_range (idx_t beg, idx_t end);
// Verify bit is less than size().
void verify_index(idx_t bit) const NOT_DEBUG_RETURN;
// Verify bit is not greater than size().
void verify_limit(idx_t bit) const NOT_DEBUG_RETURN;
// Verify [beg,end) is a valid range, e.g. beg <= end <= size().
void verify_range(idx_t beg, idx_t end) const NOT_DEBUG_RETURN;
public:
static size_t compute_size(size_t heap_size);
// Returns the amount of bytes on the heap between two marks in the bitmap.
static size_t mark_distance();
// Returns how many bytes (or bits) of the heap a single byte (or bit) of the
// mark bitmap corresponds to. This is the same as the mark distance above.
static size_t heap_map_factor() {
return mark_distance();
}
ShenandoahMarkBitMap(MemRegion heap, MemRegion storage);
// Mark word as 'strong' if it hasn't been marked strong yet.
// Return true if the word has been marked strong, false if it has already been
// marked strong or if another thread has beat us by marking it
// strong.
// Words that have been marked final before or by a concurrent thread will be
// upgraded to strong. In this case, this method also returns true.
inline bool mark_strong(HeapWord* w, bool& was_upgraded);
// Mark word as 'weak' if it hasn't been marked weak or strong yet.
// Return true if the word has been marked weak, false if it has already been
// marked strong or weak or if another thread has beat us by marking it
// strong or weak.
inline bool mark_weak(HeapWord* heap_addr);
inline bool is_marked(HeapWord* addr) const;
inline bool is_marked_strong(HeapWord* w) const;
inline bool is_marked_weak(HeapWord* addr) const;
// Return the address corresponding to the next marked bit at or after
// "addr", and before "limit", if "limit" is non-NULL. If there is no
// such bit, returns "limit" if that is non-NULL, or else "endWord()".
HeapWord* get_next_marked_addr(const HeapWord* addr,
const HeapWord* limit) const;
bm_word_t inverted_bit_mask_for_range(idx_t beg, idx_t end) const;
void clear_range_within_word (idx_t beg, idx_t end);
void clear_range (idx_t beg, idx_t end);
void clear_range_large(MemRegion mr);
void clear_range_of_words(idx_t beg, idx_t end);
void clear_large_range_of_words(idx_t beg, idx_t end);
static void clear_range_of_words(bm_word_t* map, idx_t beg, idx_t end);
};
#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKBITMAP_HPP

218
src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.inline.hpp Normal file
View File

@ -0,0 +1,218 @@
/*
* Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Red Hat, Inc. and/or its affiliates.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKBITMAP_INLINE_HPP
#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKBITMAP_INLINE_HPP
#include "gc/shenandoah/shenandoahMarkBitMap.hpp"
#include "runtime/atomic.hpp"
#include "utilities/count_trailing_zeros.hpp"
inline size_t ShenandoahMarkBitMap::address_to_index(const HeapWord* addr) const {
return (pointer_delta(addr, _covered.start()) << 1) >> _shift;
}
inline HeapWord* ShenandoahMarkBitMap::index_to_address(size_t offset) const {
return _covered.start() + ((offset >> 1) << _shift);
}
inline bool ShenandoahMarkBitMap::mark_strong(HeapWord* heap_addr, bool& was_upgraded) {
check_mark(heap_addr);
idx_t bit = address_to_index(heap_addr);
verify_index(bit);
volatile bm_word_t* const addr = word_addr(bit);
const bm_word_t mask = bit_mask(bit);
const bm_word_t mask_weak = (bm_word_t)1 << (bit_in_word(bit) + 1);
bm_word_t old_val = load_word_ordered(addr, memory_order_conservative);
do {
const bm_word_t new_val = old_val | mask;
if (new_val == old_val) {
assert(!was_upgraded, "Should be false already");
return false; // Someone else beat us to it.
}
const bm_word_t cur_val = Atomic::cmpxchg(addr, old_val, new_val, memory_order_conservative);
if (cur_val == old_val) {
was_upgraded = (cur_val & mask_weak) != 0;
return true; // Success.
}
old_val = cur_val; // The value changed, try again.
} while (true);
}
inline bool ShenandoahMarkBitMap::mark_weak(HeapWord* heap_addr) {
check_mark(heap_addr);
idx_t bit = address_to_index(heap_addr);
verify_index(bit);
volatile bm_word_t* const addr = word_addr(bit);
const bm_word_t mask_weak = (bm_word_t)1 << (bit_in_word(bit) + 1);
const bm_word_t mask_strong = (bm_word_t)1 << bit_in_word(bit);
bm_word_t old_val = load_word_ordered(addr, memory_order_conservative);
do {
if ((old_val & mask_strong) != 0) {
return false; // Already marked strong
}
const bm_word_t new_val = old_val | mask_weak;
if (new_val == old_val) {
return false; // Someone else beat us to it.
}
const bm_word_t cur_val = Atomic::cmpxchg(addr, old_val, new_val, memory_order_conservative);
if (cur_val == old_val) {
return true; // Success.
}
old_val = cur_val; // The value changed, try again.
} while (true);
}
inline bool ShenandoahMarkBitMap::is_marked_strong(HeapWord* addr) const {
check_mark(addr);
return at(address_to_index(addr));
}
inline bool ShenandoahMarkBitMap::is_marked_weak(HeapWord* addr) const {
check_mark(addr);
return at(address_to_index(addr) + 1);
}
inline bool ShenandoahMarkBitMap::is_marked(HeapWord* addr) const {
check_mark(addr);
idx_t index = address_to_index(addr);
verify_index(index);
bm_word_t mask = (bm_word_t)3 << bit_in_word(index);
return (*word_addr(index) & mask) != 0;
}
inline const ShenandoahMarkBitMap::bm_word_t ShenandoahMarkBitMap::load_word_ordered(const volatile bm_word_t* const addr, atomic_memory_order memory_order) {
if (memory_order == memory_order_relaxed || memory_order == memory_order_release) {
return Atomic::load(addr);
} else {
assert(memory_order == memory_order_acq_rel ||
memory_order == memory_order_acquire ||
memory_order == memory_order_conservative,
"unexpected memory ordering");
return Atomic::load_acquire(addr);
}
}
template<ShenandoahMarkBitMap::bm_word_t flip, bool aligned_right>
inline ShenandoahMarkBitMap::idx_t ShenandoahMarkBitMap::get_next_bit_impl(idx_t l_index, idx_t r_index) const {
STATIC_ASSERT(flip == find_ones_flip || flip == find_zeros_flip);
verify_range(l_index, r_index);
assert(!aligned_right || is_aligned(r_index, BitsPerWord), "r_index not aligned");
// The first word often contains an interesting bit, either due to
// density or because of features of the calling algorithm. So it's
// important to examine that first word with a minimum of fuss,
// minimizing setup time for later words that will be wasted if the
// first word is indeed interesting.
// The benefit from aligned_right being true is relatively small.
// It saves an operation in the setup for the word search loop.
// It also eliminates the range check on the final result.
// However, callers often have a comparison with r_index, and
// inlining often allows the two comparisons to be combined; it is
// important when !aligned_right that return paths either return
// r_index or a value dominated by a comparison with r_index.
// aligned_right is still helpful when the caller doesn't have a
// range check because features of the calling algorithm guarantee
// an interesting bit will be present.
if (l_index < r_index) {
// Get the word containing l_index, and shift out low bits.
idx_t index = to_words_align_down(l_index);
bm_word_t cword = (map(index) ^ flip) >> bit_in_word(l_index);
if ((cword & 1) != 0) {
// The first bit is similarly often interesting. When it matters
// (density or features of the calling algorithm make it likely
// the first bit is set), going straight to the next clause compares
// poorly with doing this check first; count_trailing_zeros can be
// relatively expensive, plus there is the additional range check.
// But when the first bit isn't set, the cost of having tested for
// it is relatively small compared to the rest of the search.
return l_index;
} else if (cword != 0) {
// Flipped and shifted first word is non-zero.
idx_t result = l_index + count_trailing_zeros(cword);
if (aligned_right || (result < r_index)) return result;
// Result is beyond range bound; return r_index.
} else {
// Flipped and shifted first word is zero. Word search through
// aligned up r_index for a non-zero flipped word.
idx_t limit = aligned_right
? to_words_align_down(r_index) // Miniscule savings when aligned.
: to_words_align_up(r_index);
while (++index < limit) {
cword = map(index) ^ flip;
if (cword != 0) {
idx_t result = bit_index(index) + count_trailing_zeros(cword);
if (aligned_right || (result < r_index)) return result;
// Result is beyond range bound; return r_index.
assert((index + 1) == limit, "invariant");
break;
}
}
// No bits in range; return r_index.
}
}
return r_index;
}
inline ShenandoahMarkBitMap::idx_t ShenandoahMarkBitMap::get_next_one_offset(idx_t l_offset, idx_t r_offset) const {
return get_next_bit_impl<find_ones_flip, false>(l_offset, r_offset);
}
// Returns a bit mask for a range of bits [beg, end) within a single word. Each
// bit in the mask is 0 if the bit is in the range, 1 if not in the range. The
// returned mask can be used directly to clear the range, or inverted to set the
// range. Note: end must not be 0.
inline ShenandoahMarkBitMap::bm_word_t
ShenandoahMarkBitMap::inverted_bit_mask_for_range(idx_t beg, idx_t end) const {
assert(end != 0, "does not work when end == 0");
assert(beg == end || to_words_align_down(beg) == to_words_align_down(end - 1),
"must be a single-word range");
bm_word_t mask = bit_mask(beg) - 1; // low (right) bits
if (bit_in_word(end) != 0) {
mask |= ~(bit_mask(end) - 1); // high (left) bits
}
return mask;
}
inline void ShenandoahMarkBitMap::clear_range_of_words(bm_word_t* map, idx_t beg, idx_t end) {
for (idx_t i = beg; i < end; ++i) map[i] = 0;
}
inline void ShenandoahMarkBitMap::clear_large_range_of_words(idx_t beg, idx_t end) {
assert(beg <= end, "underflow");
memset(_map + beg, 0, (end - beg) * sizeof(bm_word_t));
}
inline void ShenandoahMarkBitMap::clear_range_of_words(idx_t beg, idx_t end) {
clear_range_of_words(_map, beg, end);
}
#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHMARKBITMAP_INLINE_HPP

13
src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp
View File

@ -38,6 +38,7 @@
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
@ -129,10 +130,8 @@ void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
assert(!heap->marking_context()->is_complete(), "sanity");
// e. Abandon reference discovery and clear all discovered references.
ReferenceProcessor* rp = heap->ref_processor();
rp->disable_discovery();
ShenandoahReferenceProcessor* rp = heap->ref_processor();
rp->abandon_partial_discovery();
rp->verify_no_references_recorded();
// f. Set back forwarded objects bit back, in case some steps above dropped it.
heap->set_has_forwarded_objects(has_forwarded_objects);
@ -241,18 +240,16 @@ void ShenandoahMarkCompact::phase1_mark_heap() {
ShenandoahConcurrentMark* cm = heap->concurrent_mark();
heap->set_process_references(heap->heuristics()->can_process_references());
heap->set_unload_classes(heap->heuristics()->can_unload_classes());
ReferenceProcessor* rp = heap->ref_processor();
ShenandoahReferenceProcessor* rp = heap->ref_processor();
// enable ("weak") refs discovery
rp->enable_discovery(true /*verify_no_refs*/);
rp->setup_policy(true); // forcefully purge all soft references
rp->set_active_mt_degree(heap->workers()->active_workers());
rp->set_soft_reference_policy(true); // forcefully purge all soft references
cm->mark_roots(ShenandoahPhaseTimings::full_gc_scan_roots);
cm->finish_mark_from_roots(/* full_gc = */ true);
heap->mark_complete_marking_context();
rp->process_references(heap->workers(), false /* concurrent */);
heap->parallel_cleaning(true /* full_gc */);
}

2
src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp
View File

@ -29,11 +29,11 @@
#include "gc/shenandoah/shenandoahMarkingContext.hpp"
ShenandoahMarkingContext::ShenandoahMarkingContext(MemRegion heap_region, MemRegion bitmap_region, size_t num_regions) :
_mark_bit_map(heap_region, bitmap_region),
_top_bitmaps(NEW_C_HEAP_ARRAY(HeapWord*, num_regions, mtGC)),
_top_at_mark_starts_base(NEW_C_HEAP_ARRAY(HeapWord*, num_regions, mtGC)),
_top_at_mark_starts(_top_at_mark_starts_base -
((uintx) heap_region.start() >> ShenandoahHeapRegion::region_size_bytes_shift())) {
_mark_bit_map.initialize(heap_region, bitmap_region);
}
bool ShenandoahMarkingContext::is_bitmap_clear() const {

17
src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp
View File

@ -25,7 +25,8 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHMARKINGCONTEXT_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHMARKINGCONTEXT_HPP
#include "gc/shared/markBitMap.hpp"
#include "gc/shenandoah/shenandoahMarkBitMap.hpp"
#include "gc/shenandoah/shenandoahSharedVariables.hpp"
#include "memory/allocation.hpp"
#include "memory/memRegion.hpp"
#include "oops/oopsHierarchy.hpp"
@ -35,7 +36,7 @@
*/
class ShenandoahMarkingContext : public CHeapObj<mtGC> {
private:
MarkBitMap _mark_bit_map;
ShenandoahMarkBitMap _mark_bit_map;
HeapWord** const _top_bitmaps;
HeapWord** const _top_at_mark_starts_base;
@ -51,15 +52,19 @@ public:
* been marked by this thread. Returns false if the object has already been marked,
* or if a competing thread succeeded in marking this object.
*/
inline bool mark(oop obj);
inline bool mark_strong(oop obj, bool& was_upgraded);
inline bool mark_weak(oop obj);
inline bool is_marked(oop obj) const;
// Simple versions of marking accessors, to be used outside of marking (e.g. no possible concurrent updates)
inline bool is_marked(oop) const;
inline bool is_marked_strong(oop obj) const;
inline bool is_marked_weak(oop obj) const;
inline HeapWord* get_next_marked_addr(HeapWord* addr, HeapWord* limit) const;
inline bool allocated_after_mark_start(oop obj) const;
inline bool allocated_after_mark_start(HeapWord* addr) const;
inline MarkBitMap* mark_bit_map();
inline HeapWord* top_at_mark_start(ShenandoahHeapRegion* r) const;
inline void capture_top_at_mark_start(ShenandoahHeapRegion* r);
inline void reset_top_at_mark_start(ShenandoahHeapRegion* r);

24
src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp
View File

@ -25,19 +25,33 @@
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHMARKINGCONTEXT_INLINE_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHMARKINGCONTEXT_INLINE_HPP
#include "gc/shenandoah/shenandoahMarkBitMap.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.hpp"
inline MarkBitMap* ShenandoahMarkingContext::mark_bit_map() {
return &_mark_bit_map;
inline bool ShenandoahMarkingContext::mark_strong(oop obj, bool& was_upgraded) {
shenandoah_assert_not_forwarded(NULL, obj);
return (! allocated_after_mark_start(obj)) && _mark_bit_map.mark_strong(cast_from_oop<HeapWord*>(obj), was_upgraded);
}
inline bool ShenandoahMarkingContext::mark(oop obj) {
inline bool ShenandoahMarkingContext::mark_weak(oop obj) {
shenandoah_assert_not_forwarded(NULL, obj);
return (! allocated_after_mark_start(obj)) && _mark_bit_map.par_mark(obj);
return (! allocated_after_mark_start(obj)) && _mark_bit_map.mark_weak(cast_from_oop<HeapWord *>(obj));
}
inline bool ShenandoahMarkingContext::is_marked(oop obj) const {
return allocated_after_mark_start(obj) || _mark_bit_map.is_marked(obj);
return allocated_after_mark_start(obj) || _mark_bit_map.is_marked(cast_from_oop<HeapWord *>(obj));
}
inline bool ShenandoahMarkingContext::is_marked_strong(oop obj) const {
return allocated_after_mark_start(obj) || _mark_bit_map.is_marked_strong(cast_from_oop<HeapWord*>(obj));
}
inline bool ShenandoahMarkingContext::is_marked_weak(oop obj) const {
return allocated_after_mark_start(obj) || _mark_bit_map.is_marked_weak(cast_from_oop<HeapWord *>(obj));
}
inline HeapWord* ShenandoahMarkingContext::get_next_marked_addr(HeapWord* start, HeapWord* limit) const {
return _mark_bit_map.get_next_marked_addr(start, limit);
}
inline bool ShenandoahMarkingContext::allocated_after_mark_start(oop obj) const {

29
src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp
View File

@ -49,13 +49,22 @@ private:
ShenandoahObjToScanQueue* _queue;
ShenandoahHeap* _heap;
ShenandoahMarkingContext* const _mark_context;
bool _weak;
protected:
template <class T, UpdateRefsMode UPDATE_MODE, StringDedupMode STRING_DEDUP>
void work(T *p);
public:
ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp);
ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp);
bool is_weak() const {
return _weak;
}
void set_weak(bool weak) {
_weak = weak;
}
};
class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkRefsSuperClosure {
@ -64,7 +73,7 @@ private:
inline void do_oop_work(T* p) { work<T, CONCURRENT, NO_DEDUP>(p); }
public:
ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
@ -78,7 +87,7 @@ private:
inline void do_oop_work(T* p) { work<T, CONCURRENT, ENQUEUE_DEDUP>(p); }
public:
ShenandoahMarkUpdateRefsDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkUpdateRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
@ -92,7 +101,7 @@ private:
inline void do_oop_work(T* p) { work<T, CONCURRENT, NO_DEDUP>(p); }
public:
ShenandoahMarkUpdateRefsMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkUpdateRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
@ -106,7 +115,7 @@ private:
inline void do_oop_work(T* p) { work<T, CONCURRENT, ENQUEUE_DEDUP>(p); }
public:
ShenandoahMarkUpdateRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkUpdateRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
@ -120,7 +129,7 @@ private:
inline void do_oop_work(T* p) { work<T, NONE, NO_DEDUP>(p); }
public:
ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
@ -134,7 +143,7 @@ private:
inline void do_oop_work(T* p) { work<T, NONE, ENQUEUE_DEDUP>(p); }
public:
ShenandoahMarkRefsDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
@ -148,7 +157,7 @@ private:
inline void do_oop_work(T* p) { work<T, RESOLVE, NO_DEDUP>(p); }
public:
ShenandoahMarkResolveRefsClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkResolveRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
@ -162,7 +171,7 @@ private:
inline void do_oop_work(T* p) { work<T, NONE, NO_DEDUP>(p); }
public:
ShenandoahMarkRefsMetadataClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }
@ -176,7 +185,7 @@ private:
inline void do_oop_work(T* p) { work<T, NONE, ENQUEUE_DEDUP>(p); }
public:
ShenandoahMarkRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ReferenceProcessor* rp) :
ShenandoahMarkRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) :
ShenandoahMarkRefsSuperClosure(q, rp) {};
virtual void do_oop(narrowOop* p) { do_oop_work(p); }

2
src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp
View File

@ -30,7 +30,7 @@
template<class T, UpdateRefsMode UPDATE_REFS, StringDedupMode STRING_DEDUP>
inline void ShenandoahMarkRefsSuperClosure::work(T *p) {
ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, STRING_DEDUP>(p, _heap, _queue, _mark_context);
ShenandoahConcurrentMark::mark_through_ref<T, UPDATE_REFS, STRING_DEDUP>(p, _heap, _queue, _mark_context, _weak);
}
template <class T>

10
src/hotspot/share/gc/shenandoah/shenandoahPacer.cpp
View File

@ -153,16 +153,6 @@ void ShenandoahPacer::setup_for_idle() {
* the allocators unnecessarily, allow them to run unimpeded.
*/
void ShenandoahPacer::setup_for_preclean() {
assert(ShenandoahPacing, "Only be here when pacing is enabled");
size_t initial = _heap->max_capacity();
restart_with(initial, 1.0);
log_info(gc, ergo)("Pacer for Precleaning. Non-Taxable: " SIZE_FORMAT "%s",
byte_size_in_proper_unit(initial), proper_unit_for_byte_size(initial));
}
void ShenandoahPacer::setup_for_reset() {
assert(ShenandoahPacing, "Only be here when pacing is enabled");

1
src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp
View File

@ -79,7 +79,6 @@ public:
void setup_for_updaterefs();
void setup_for_reset();
void setup_for_preclean();
inline void report_mark(size_t words);
inline void report_evac(size_t words);

1
src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp
View File

@ -113,6 +113,7 @@ bool ShenandoahPhaseTimings::is_worker_phase(Phase phase) {
case heap_iteration_roots:
case conc_mark_roots:
case conc_weak_roots_work:
case conc_weak_refs_work:
case conc_strong_roots:
return true;
default:

5
src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp
View File

@ -60,8 +60,6 @@ class outputStream;
f(conc_mark_roots, " Roots ") \
SHENANDOAH_PAR_PHASE_DO(conc_mark_roots, " CM: ", f) \
\
f(conc_preclean, "Concurrent Precleaning") \
\
f(final_mark_gross, "Pause Final Mark (G)") \
f(final_mark, "Pause Final Mark (N)") \
f(update_roots, " Update Roots") \
@ -82,6 +80,9 @@ class outputStream;
f(init_evac, " Initial Evacuation") \
SHENANDOAH_PAR_PHASE_DO(evac_, " E: ", f) \
\
f(conc_weak_refs, "Concurrent Weak References") \
f(conc_weak_refs_work, " Process") \
SHENANDOAH_PAR_PHASE_DO(conc_weak_refs_work_, " CWRF: ", f) \
f(conc_weak_roots, "Concurrent Weak Roots") \
f(conc_weak_roots_work, " Roots") \
SHENANDOAH_PAR_PHASE_DO(conc_weak_roots_work_, " CWR: ", f) \

592
src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp Normal file
View File

@ -0,0 +1,592 @@
/*
* Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Red Hat, Inc. and/or its affiliates.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "classfile/javaClasses.hpp"
#include "gc/shenandoah/shenandoahOopClosures.hpp"
#include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/shenandoahUtils.hpp"
#include "runtime/atomic.hpp"
#include "logging/log.hpp"
static ReferenceType reference_type(oop reference) {
return InstanceKlass::cast(reference->klass())->reference_type();
}
static const char* reference_type_name(ReferenceType type) {
switch (type) {
case REF_SOFT:
return "Soft";
case REF_WEAK:
return "Weak";
case REF_FINAL:
return "Final";
case REF_PHANTOM:
return "Phantom";
default:
ShouldNotReachHere();
return NULL;
}
}
template <typename T>
static void set_oop_field(T* field, oop value);
template <>
void set_oop_field<oop>(oop* field, oop value) {
*field = value;
}
template <>
void set_oop_field<narrowOop>(narrowOop* field, oop value) {
*field = CompressedOops::encode(value);
}
static oop lrb(oop obj) {
if (obj != NULL && ShenandoahHeap::heap()->marking_context()->is_marked(obj)) {
return ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj);
} else {
return obj;
}
}
template <typename T>
static volatile T* reference_referent_addr(oop reference) {
return (volatile T*)java_lang_ref_Reference::referent_addr_raw(reference);
}
template <typename T>
static oop reference_referent(oop reference) {
T heap_oop = Atomic::load(reference_referent_addr<T>(reference));
return CompressedOops::decode(heap_oop);
}
static void reference_set_referent(oop reference, oop referent) {
java_lang_ref_Reference::set_referent_raw(reference, referent);
}
template <typename T>
static T* reference_discovered_addr(oop reference) {
return reinterpret_cast<T*>(java_lang_ref_Reference::discovered_addr_raw(reference));
}
template <typename T>
static oop reference_discovered(oop reference) {
T heap_oop = *reference_discovered_addr<T>(reference);
return lrb(CompressedOops::decode(heap_oop));
}
template <typename T>
static void reference_set_discovered(oop reference, oop discovered);
template <>
void reference_set_discovered<oop>(oop reference, oop discovered) {
*reference_discovered_addr<oop>(reference) = discovered;
}
template <>
void reference_set_discovered<narrowOop>(oop reference, oop discovered) {
*reference_discovered_addr<narrowOop>(reference) = CompressedOops::encode(discovered);
}
template<typename T>
static bool reference_cas_discovered(oop reference, oop discovered);
template<>
bool reference_cas_discovered<narrowOop>(oop reference, oop discovered) {
volatile narrowOop* addr = reinterpret_cast<volatile narrowOop*>(java_lang_ref_Reference::discovered_addr_raw(reference));
narrowOop compare = CompressedOops::encode(NULL);
narrowOop exchange = CompressedOops::encode(discovered);
return Atomic::cmpxchg(addr, compare, exchange) == compare;
}
template<>
bool reference_cas_discovered<oop>(oop reference, oop discovered) {
volatile oop* addr = reinterpret_cast<volatile oop*>(java_lang_ref_Reference::discovered_addr_raw(reference));
return Atomic::cmpxchg(addr, oop(NULL), discovered) == NULL;
}
template <typename T>
static T* reference_next_addr(oop reference) {
return reinterpret_cast<T*>(java_lang_ref_Reference::next_addr_raw(reference));
}
template <typename T>
static oop reference_next(oop reference) {
T heap_oop = RawAccess<>::oop_load(reference_next_addr<T>(reference));
return lrb(CompressedOops::decode(heap_oop));
}
static void reference_set_next(oop reference, oop next) {
java_lang_ref_Reference::set_next_raw(reference, next);
}
static void soft_reference_update_clock() {
const jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC;
java_lang_ref_SoftReference::set_clock(now);
}
ShenandoahRefProcThreadLocal::ShenandoahRefProcThreadLocal() :
_discovered_list(NULL),
_encountered_count(),
_discovered_count(),
_enqueued_count() {
}
void ShenandoahRefProcThreadLocal::reset() {
_discovered_list = NULL;
_mark_closure = NULL;
for (uint i = 0; i < reference_type_count; i++) {
_encountered_count[i] = 0;
_discovered_count[i] = 0;
_enqueued_count[i] = 0;
}
}
template <typename T>
T* ShenandoahRefProcThreadLocal::discovered_list_addr() {
return reinterpret_cast<T*>(&_discovered_list);
}
template <>
oop ShenandoahRefProcThreadLocal::discovered_list_head<oop>() const {
return *reinterpret_cast<const oop*>(&_discovered_list);
}
template <>
oop ShenandoahRefProcThreadLocal::discovered_list_head<narrowOop>() const {
return CompressedOops::decode(*reinterpret_cast<const narrowOop*>(&_discovered_list));
}
template <>
void ShenandoahRefProcThreadLocal::set_discovered_list_head<narrowOop>(oop head) {
*discovered_list_addr<narrowOop>() = CompressedOops::encode(head);
}
template <>
void ShenandoahRefProcThreadLocal::set_discovered_list_head<oop>(oop head) {
*discovered_list_addr<oop>() = head;
}
ShenandoahReferenceProcessor::ShenandoahReferenceProcessor(uint max_workers) :
_soft_reference_policy(NULL),
_ref_proc_thread_locals(NEW_C_HEAP_ARRAY(ShenandoahRefProcThreadLocal, max_workers, mtGC)),
_pending_list(NULL),
_pending_list_tail(&_pending_list),
_iterate_discovered_list_id(0U) {
for (size_t i = 0; i < max_workers; i++) {
_ref_proc_thread_locals[i].reset();
}
}
void ShenandoahReferenceProcessor::reset_thread_locals() {
uint max_workers = ShenandoahHeap::heap()->max_workers();
for (uint i = 0; i < max_workers; i++) {
_ref_proc_thread_locals[i].reset();
}
}
void ShenandoahReferenceProcessor::set_mark_closure(uint worker_id, ShenandoahMarkRefsSuperClosure* mark_closure) {
_ref_proc_thread_locals[worker_id].set_mark_closure(mark_closure);
}
void ShenandoahReferenceProcessor::set_soft_reference_policy(bool clear) {
static AlwaysClearPolicy always_clear_policy;
static LRUMaxHeapPolicy lru_max_heap_policy;
if (clear) {
log_info(gc, ref)("Clearing All SoftReferences");
_soft_reference_policy = &always_clear_policy;
} else {
_soft_reference_policy = &lru_max_heap_policy;
}
_soft_reference_policy->setup();
}
template <typename T>
bool ShenandoahReferenceProcessor::is_inactive(oop reference, oop referent, ReferenceType type) const {
if (type == REF_FINAL) {
// A FinalReference is inactive if its next field is non-null. An application can't
// call enqueue() or clear() on a FinalReference.
return reference_next<T>(reference) != NULL;
} else {
// A non-FinalReference is inactive if the referent is null. The referent can only
// be null if the application called Reference.enqueue() or Reference.clear().
return referent == NULL;
}
}
bool ShenandoahReferenceProcessor::is_strongly_live(oop referent) const {
return ShenandoahHeap::heap()->marking_context()->is_marked_strong(referent);
}
bool ShenandoahReferenceProcessor::is_softly_live(oop reference, ReferenceType type) const {
if (type != REF_SOFT) {
// Not a SoftReference
return false;
}
// Ask SoftReference policy
const jlong clock = java_lang_ref_SoftReference::clock();
assert(clock != 0, "Clock not initialized");
assert(_soft_reference_policy != NULL, "Policy not initialized");
return !_soft_reference_policy->should_clear_reference(reference, clock);
}
template <typename T>
bool ShenandoahReferenceProcessor::should_discover(oop reference, ReferenceType type) const {
T* referent_addr = (T*) java_lang_ref_Reference::referent_addr_raw(reference);
T heap_oop = RawAccess<>::oop_load(referent_addr);
oop referent = CompressedOops::decode_not_null(heap_oop);
if (is_inactive<T>(reference, referent, type)) {
log_trace(gc,ref)("Reference inactive: " PTR_FORMAT, p2i(reference));
return false;
}
if (is_strongly_live(referent)) {
log_trace(gc,ref)("Reference strongly live: " PTR_FORMAT, p2i(reference));
return false;
}
if (is_softly_live(reference, type)) {
log_trace(gc,ref)("Reference softly live: " PTR_FORMAT, p2i(reference));
return false;
}
return true;
}
template <typename T>
bool ShenandoahReferenceProcessor::should_drop(oop reference, ReferenceType type) const {
const oop referent = reference_referent<T>(reference);
if (referent == NULL) {
// Reference has been cleared, by a call to Reference.enqueue()
// or Reference.clear() from the application, which means we
// should drop the reference.
return true;
}
// Check if the referent is still alive, in which case we should
// drop the reference.
if (type == REF_PHANTOM) {
return ShenandoahHeap::heap()->complete_marking_context()->is_marked(referent);
} else {
return ShenandoahHeap::heap()->complete_marking_context()->is_marked_strong(referent);
}
}
template <typename T>
void ShenandoahReferenceProcessor::make_inactive(oop reference, ReferenceType type) const {
if (type == REF_FINAL) {
// Don't clear referent. It is needed by the Finalizer thread to make the call
// to finalize(). A FinalReference is instead made inactive by self-looping the
// next field. An application can't call FinalReference.enqueue(), so there is
// no race to worry about when setting the next field.
assert(reference_next<T>(reference) == NULL, "Already inactive");
assert(ShenandoahHeap::heap()->marking_context()->is_marked(reference_referent<T>(reference)), "only make inactive final refs with alive referents");
reference_set_next(reference, reference);
} else {
// Clear referent
reference_set_referent(reference, NULL);
}
}
template <typename T>
bool ShenandoahReferenceProcessor::discover(oop reference, ReferenceType type, uint worker_id) {
if (!should_discover<T>(reference, type)) {
// Not discovered
return false;
}
if (reference_discovered<T>(reference) != NULL) {
// Already discovered. This can happen if the reference is marked finalizable first, and then strong,
// in which case it will be seen 2x by marking.
log_trace(gc,ref)("Reference already discovered: " PTR_FORMAT, p2i(reference));
return true;
}
if (type == REF_FINAL) {
ShenandoahMarkRefsSuperClosure* cl = _ref_proc_thread_locals[worker_id].mark_closure();
bool weak = cl->is_weak();
cl->set_weak(true);
if (UseCompressedOops) {
cl->do_oop(reinterpret_cast<narrowOop*>(java_lang_ref_Reference::referent_addr_raw(reference)));
} else {
cl->do_oop(reinterpret_cast<oop*>(java_lang_ref_Reference::referent_addr_raw(reference)));
}
cl->set_weak(weak);
}
// Add reference to discovered list
assert(worker_id != ShenandoahThreadLocalData::INVALID_WORKER_ID, "need valid worker ID");
ShenandoahRefProcThreadLocal& refproc_data = _ref_proc_thread_locals[worker_id];
oop discovered_head = refproc_data.discovered_list_head<T>();
if (discovered_head == NULL) {
// Self-loop tail of list. We distinguish discovered from not-discovered references by looking at their
// discovered field: if it is NULL, then it is not-yet discovered, otherwise it is discovered
discovered_head = reference;
}
if (reference_cas_discovered<T>(reference, discovered_head)) {
refproc_data.set_discovered_list_head<T>(reference);
assert(refproc_data.discovered_list_head<T>() == reference, "reference must be new discovered head");
log_trace(gc, ref)("Discovered Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
_ref_proc_thread_locals[worker_id].inc_discovered(type);
}
return true;
}
bool ShenandoahReferenceProcessor::discover_reference(oop reference, ReferenceType type) {
if (!RegisterReferences) {
// Reference processing disabled
return false;
}
log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
uint worker_id = ShenandoahThreadLocalData::worker_id(Thread::current());
_ref_proc_thread_locals->inc_encountered(type);
if (UseCompressedOops) {
return discover<narrowOop>(reference, type, worker_id);
} else {
return discover<oop>(reference, type, worker_id);
}
}
template <typename T>
oop ShenandoahReferenceProcessor::drop(oop reference, ReferenceType type) {
log_trace(gc, ref)("Dropped Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
assert(reference_referent<T>(reference) == NULL ||
ShenandoahHeap::heap()->marking_context()->is_marked(reference_referent<T>(reference)), "only drop references with alive referents");
// Unlink and return next in list
oop next = reference_discovered<T>(reference);
reference_set_discovered<T>(reference, NULL);
return next;
}
template <typename T>
T* ShenandoahReferenceProcessor::keep(oop reference, ReferenceType type, uint worker_id) {
log_trace(gc, ref)("Enqueued Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
// Update statistics
_ref_proc_thread_locals[worker_id].inc_enqueued(type);
// Make reference inactive
make_inactive<T>(reference, type);
// Return next in list
return reference_discovered_addr<T>(reference);
}
template <typename T>
void ShenandoahReferenceProcessor::process_references(ShenandoahRefProcThreadLocal& refproc_data, uint worker_id) {;
log_trace(gc, ref)("Processing discovered list #%u : " PTR_FORMAT, worker_id, p2i(refproc_data.discovered_list_head<T>()));
T* list = refproc_data.discovered_list_addr<T>();
// The list head is basically a GC root, we need to resolve and update it,
// otherwise we will later swap a from-space ref into Universe::pending_list().
if (!CompressedOops::is_null(*list)) {
oop first_resolved = lrb(CompressedOops::decode_not_null(*list));
set_oop_field(list, first_resolved);
}
T* p = list;
while (true) {
const oop reference = lrb(CompressedOops::decode(*p));
if (reference == NULL) {
break;
}
log_trace(gc, ref)("Processing reference: " PTR_FORMAT, p2i(reference));
const ReferenceType type = reference_type(reference);
if (should_drop<T>(reference, type)) {
set_oop_field(p, drop<T>(reference, type));
} else {
p = keep<T>(reference, type, worker_id);
}
const oop discovered = lrb(reference_discovered<T>(reference));
if (reference == discovered) {
// Reset terminating self-loop to NULL
reference_set_discovered<T>(reference, oop(NULL));
break;
}
}
// Prepend discovered references to internal pending list
if (!CompressedOops::is_null(*list)) {
oop head = lrb(CompressedOops::decode_not_null(*list));
shenandoah_assert_not_in_cset_except(&head, head, ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahLoadRefBarrier);
oop prev = Atomic::xchg(&_pending_list, head);
RawAccess<>::oop_store(p, prev);
if (prev == NULL) {
// First to prepend to list, record tail
_pending_list_tail = reinterpret_cast<void*>(p);
}
// Clear discovered list
set_oop_field(list, oop(NULL));
}
}
void ShenandoahReferenceProcessor::work() {
// Process discovered references
uint max_workers = ShenandoahHeap::heap()->max_workers();
uint worker_id = Atomic::add(&_iterate_discovered_list_id, 1U) - 1;
while (worker_id < max_workers) {
if (UseCompressedOops) {
process_references<narrowOop>(_ref_proc_thread_locals[worker_id], worker_id);
} else {
process_references<oop>(_ref_proc_thread_locals[worker_id], worker_id);
}
worker_id = Atomic::add(&_iterate_discovered_list_id, 1U) - 1;
}
}
class ShenandoahReferenceProcessorTask : public AbstractGangTask {
private:
ShenandoahReferenceProcessor* const _reference_processor;
public:
ShenandoahReferenceProcessorTask(ShenandoahReferenceProcessor* reference_processor) :
AbstractGangTask("ShenandoahReferenceProcessorTask"),
_reference_processor(reference_processor) {
}
virtual void work(uint worker_id) {
ShenandoahConcurrentWorkerSession worker_session(worker_id);
_reference_processor->work();
}
};
void ShenandoahReferenceProcessor::process_references(WorkGang* workers, bool concurrent) {
Atomic::release_store_fence(&_iterate_discovered_list_id, 0U);
// Process discovered lists
ShenandoahReferenceProcessorTask task(this);
workers->run_task(&task);
// Update SoftReference clock
soft_reference_update_clock();
// Collect, log and trace statistics
collect_statistics();
enqueue_references(concurrent);
}
void ShenandoahReferenceProcessor::enqueue_references_locked() {
// Prepend internal pending list to external pending list
shenandoah_assert_not_in_cset_except(&_pending_list, _pending_list, ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahLoadRefBarrier);
if (UseCompressedOops) {
*reinterpret_cast<narrowOop*>(_pending_list_tail) = CompressedOops::encode(Universe::swap_reference_pending_list(_pending_list));
} else {
*reinterpret_cast<oop*>(_pending_list_tail) = Universe::swap_reference_pending_list(_pending_list);
}
}
void ShenandoahReferenceProcessor::enqueue_references(bool concurrent) {
if (_pending_list == NULL) {
// Nothing to enqueue
return;
}
if (!concurrent) {
// When called from mark-compact or degen-GC, the locking is done by the VMOperation,
enqueue_references_locked();
} else {
// Heap_lock protects external pending list
MonitorLocker ml(Heap_lock, Mutex::_no_safepoint_check_flag);
enqueue_references_locked();
// Notify ReferenceHandler thread
ml.notify_all();
}
// Reset internal pending list
_pending_list = NULL;
_pending_list_tail = &_pending_list;
}
template<typename T>
void ShenandoahReferenceProcessor::clean_discovered_list(T* list) {
T discovered = *list;
while (!CompressedOops::is_null(discovered)) {
oop discovered_ref = CompressedOops::decode_not_null(discovered);
set_oop_field<T>(list, oop(NULL));
list = reference_discovered_addr<T>(discovered_ref);
discovered = *list;
}
}
void ShenandoahReferenceProcessor::abandon_partial_discovery() {
uint max_workers = ShenandoahHeap::heap()->max_workers();
for (uint index = 0; index < max_workers; index++) {
if (UseCompressedOops) {
clean_discovered_list<narrowOop>(_ref_proc_thread_locals[index].discovered_list_addr<narrowOop>());
} else {
clean_discovered_list<oop>(_ref_proc_thread_locals[index].discovered_list_addr<oop>());
}
}
if (_pending_list != NULL) {
oop pending = _pending_list;
_pending_list = NULL;
if (UseCompressedOops) {
narrowOop* list = reference_discovered_addr<narrowOop>(pending);
clean_discovered_list<narrowOop>(list);
} else {
oop* list = reference_discovered_addr<oop>(pending);
clean_discovered_list<oop>(list);
}
}
_pending_list_tail = &_pending_list;
}
void ShenandoahReferenceProcessor::collect_statistics() {
Counters encountered = {};
Counters discovered = {};
Counters enqueued = {};
uint max_workers = ShenandoahHeap::heap()->max_workers();
for (uint i = 0; i < max_workers; i++) {
for (size_t type = 0; type < reference_type_count; type++) {
encountered[type] += _ref_proc_thread_locals[i].encountered((ReferenceType)type);
discovered[type] += _ref_proc_thread_locals[i].discovered((ReferenceType)type);
enqueued[type] += _ref_proc_thread_locals[i].enqueued((ReferenceType)type);
}
}
log_info(gc,ref)("Encountered references: Soft: " SIZE_FORMAT ", Weak: " SIZE_FORMAT ", Final: " SIZE_FORMAT ", Phantom: " SIZE_FORMAT,
encountered[REF_SOFT], encountered[REF_WEAK], encountered[REF_FINAL], encountered[REF_PHANTOM]);
log_info(gc,ref)("Discovered references: Soft: " SIZE_FORMAT ", Weak: " SIZE_FORMAT ", Final: " SIZE_FORMAT ", Phantom: " SIZE_FORMAT,
discovered[REF_SOFT], discovered[REF_WEAK], discovered[REF_FINAL], discovered[REF_PHANTOM]);
log_info(gc,ref)("Enqueued references: Soft: " SIZE_FORMAT ", Weak: " SIZE_FORMAT ", Final: " SIZE_FORMAT ", Phantom: " SIZE_FORMAT,
enqueued[REF_SOFT], enqueued[REF_WEAK], enqueued[REF_FINAL], enqueued[REF_PHANTOM]);
}

185
src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.hpp Normal file
View File

@ -0,0 +1,185 @@
/*
* Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, Red Hat, Inc. and/or its affiliates.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHREFERENCEPROCESSOR_HPP
#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHREFERENCEPROCESSOR_HPP
#include "gc/shared/referenceDiscoverer.hpp"
#include "memory/allocation.hpp"
class ShenandoahMarkRefsSuperClosure;
class WorkGang;
static const size_t reference_type_count = REF_PHANTOM + 1;
typedef size_t Counters[reference_type_count];
/*
* Shenandoah concurrent reference processing
*
* Concurrent reference processing is made up of two main phases:
* 1. Concurrent reference marking: Discover all j.l.r.Reference objects and determine reachability of all live objects.
* 2. Concurrent reference processing: For all discoved j.l.r.References, determine whether to keep them alive or clean
* them. Also, clean and enqueue relevant references concurrently.
*
* Concurrent reference marking:
* The goal here is to establish the kind of reachability for all objects on the heap. We distinguish two kinds of
* reachability:
* - An object is 'strongly reachable' if it can be found by searching transitively from GC roots.
* - An object is 'finalizably reachable' if it is not strongly reachable, but can be found by searching
* from the referents of FinalReferences.
*
* These reachabilities are implemented in shenandoahMarkBitMap.*
* Conceptually, marking starts with a strong wavefront at the GC roots. Whenever a Reference object is encountered,
* it may be discovered by the ShenandoahReferenceProcessor. If it is discovered, it
* gets added to the discovered list, and that wavefront stops there, except when it's a FinalReference, in which
* case the wavefront switches to finalizable marking and marks through the referent. When a Reference is not
* discovered, e.g. if it's a SoftReference that is not eligible for discovery, then marking continues as if the
* Reference was a regular object. Whenever a strong wavefront encounters an object that is already marked
* finalizable, then the object's reachability is upgraded to strong.
*
* Concurrent reference processing:
* This happens after the concurrent marking phase and the final marking pause, when reachability for all objects
* has been established.
* The discovered list is scanned and for each reference is decided what to do:
* - If the referent is reachable (finalizable for PhantomReference, strong for all others), then the Reference
* is dropped from the discovered list and otherwise ignored
* - Otherwise its referent becomes cleared and the Reference added to the pending list, from which it will later
* be processed (e.g. enqueued in its ReferenceQueue) by the Java ReferenceHandler thread.
*
* In order to prevent resurrection by Java threads calling Reference.get() concurrently while we are clearing
* referents, we employ a special barrier, the native LRB, which returns NULL when the referent is unreachable.
*/
class ShenandoahRefProcThreadLocal : public CHeapObj<mtGC> {
private:
void* _discovered_list;
ShenandoahMarkRefsSuperClosure* _mark_closure;
Counters _encountered_count;
Counters _discovered_count;
Counters _enqueued_count;
public:
ShenandoahRefProcThreadLocal();
ShenandoahRefProcThreadLocal(const ShenandoahRefProcThreadLocal&) = delete; // non construction-copyable
ShenandoahRefProcThreadLocal& operator=(const ShenandoahRefProcThreadLocal&) = delete; // non copyable
void reset();
ShenandoahMarkRefsSuperClosure* mark_closure() const {
return _mark_closure;
}
void set_mark_closure(ShenandoahMarkRefsSuperClosure* mark_closure) {
_mark_closure = mark_closure;
}
template<typename T>
T* discovered_list_addr();
template<typename T>
oop discovered_list_head() const;
template<typename T>
void set_discovered_list_head(oop head);
size_t encountered(ReferenceType type) const {
return _encountered_count[type];
}
size_t discovered(ReferenceType type) const {
return _discovered_count[type];
}
size_t enqueued(ReferenceType type) const {
return _enqueued_count[type];
}
void inc_encountered(ReferenceType type) {
_encountered_count[type]++;
}
void inc_discovered(ReferenceType type) {
_discovered_count[type]++;
}
void inc_enqueued(ReferenceType type) {
_enqueued_count[type]++;
}
};
class ShenandoahReferenceProcessor : public ReferenceDiscoverer {
private:
ReferencePolicy* _soft_reference_policy;
ShenandoahRefProcThreadLocal* _ref_proc_thread_locals;
oop _pending_list;
void* _pending_list_tail; // T*
volatile uint _iterate_discovered_list_id;
template <typename T>
bool is_inactive(oop reference, oop referent, ReferenceType type) const;
bool is_strongly_live(oop referent) const;
bool is_softly_live(oop reference, ReferenceType type) const;
template <typename T>
bool should_discover(oop reference, ReferenceType type) const;
template <typename T>
bool should_drop(oop reference, ReferenceType type) const;
template <typename T>
void make_inactive(oop reference, ReferenceType type) const;
template <typename T>
bool discover(oop reference, ReferenceType type, uint worker_id);
template <typename T>
oop drop(oop reference, ReferenceType type);
template <typename T>
T* keep(oop reference, ReferenceType type, uint worker_id);
template <typename T>
void process_references(ShenandoahRefProcThreadLocal& refproc_data, uint worker_id);
void enqueue_references_locked();
void enqueue_references(bool concurrent);
void collect_statistics();
template<typename T>
void clean_discovered_list(T* list);
public:
ShenandoahReferenceProcessor(uint max_workers);
void reset_thread_locals();
void set_mark_closure(uint worker_id, ShenandoahMarkRefsSuperClosure* mark_closure);
void set_soft_reference_policy(bool clear);
bool discover_reference(oop obj, ReferenceType type) override;
void process_references(WorkGang* workers, bool concurrent);
void work();
void abandon_partial_discovery();
};
#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHREFERENCEPROCESSOR_HPP

4
src/hotspot/share/gc/shenandoah/shenandoahRuntime.cpp
View File

@ -69,3 +69,7 @@ JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak(oopDesc * src, oop* load_addr))
return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<ON_UNKNOWN_OOP_REF, oop>(oop(src), load_addr);
JRT_END
JRT_LEAF(oopDesc*, ShenandoahRuntime::load_reference_barrier_weak_narrow(oopDesc * src, narrowOop* load_addr))
return (oopDesc*) ShenandoahBarrierSet::barrier_set()->load_reference_barrier<ON_UNKNOWN_OOP_REF, narrowOop>(oop(src), load_addr);
JRT_END

1
src/hotspot/share/gc/shenandoah/shenandoahRuntime.hpp
View File

@ -42,6 +42,7 @@ public:
static oopDesc* load_reference_barrier_narrow(oopDesc* src, narrowOop* load_addr);
static oopDesc* load_reference_barrier_weak(oopDesc* src, oop* load_addr);
static oopDesc* load_reference_barrier_weak_narrow(oopDesc* src, narrowOop* load_addr);
static void shenandoah_clone_barrier(oopDesc* src);
};

55
src/hotspot/share/gc/shenandoah/shenandoahTaskqueue.hpp
View File

@ -74,11 +74,15 @@ private:
// that the block has the size of 2^pow. This requires for pow to have only 5 bits (2^32) to encode
// all possible arrays.
//
// |---------oop---------|-pow-|--chunk---|
// |xx-------oop---------|-pow-|--chunk---|
// 0 49 54 64
//
// By definition, chunk == 0 means "no chunk", i.e. chunking starts from 1.
//
// Lower bits of oop are reserved to handle "skip_live" and "strong" properties. Since this encoding
// stores uncompressed oops, those bits are always available. These bits default to zero for "skip_live"
// and "weak". This aligns with their frequent values: strong/counted-live references.
//
// This encoding gives a few interesting benefits:
//
// a) Encoding/decoding regular oops is very simple, because the upper bits are zero in that task:
@ -145,7 +149,9 @@ private:
static const uint8_t pow_shift = oop_bits;
static const uint8_t chunk_shift = oop_bits + pow_bits;
static const uintptr_t oop_extract_mask = right_n_bits(oop_bits);
static const uintptr_t oop_extract_mask = right_n_bits(oop_bits) - 3;
static const uintptr_t skip_live_extract_mask = 1 << 0;
static const uintptr_t weak_extract_mask = 1 << 1;
static const uintptr_t chunk_pow_extract_mask = ~right_n_bits(oop_bits);
static const int chunk_range_mask = right_n_bits(chunk_bits);
@ -169,9 +175,24 @@ private:
return (int) ((val >> pow_shift) & pow_range_mask);
}
inline uintptr_t encode_oop(oop obj) const {
inline bool decode_weak(uintptr_t val) const {
return (val & weak_extract_mask) != 0;
}
inline bool decode_cnt_live(uintptr_t val) const {
return (val & skip_live_extract_mask) == 0;
}
inline uintptr_t encode_oop(oop obj, bool skip_live, bool weak) const {
STATIC_ASSERT(oop_shift == 0);
return cast_from_oop<uintptr_t>(obj);
uintptr_t encoded = cast_from_oop<uintptr_t>(obj);
if (skip_live) {
encoded |= skip_live_extract_mask;
}
if (weak) {
encoded |= weak_extract_mask;
}
return encoded;
}
inline uintptr_t encode_chunk(int chunk) const {
@ -183,19 +204,23 @@ private:
}
public:
ShenandoahMarkTask(oop o = NULL) {
uintptr_t enc = encode_oop(o);
assert(decode_oop(enc) == o, "oop encoding should work: " PTR_FORMAT, p2i(o));
assert(decode_not_chunked(enc), "task should not be chunked");
ShenandoahMarkTask(oop o = NULL, bool skip_live = false, bool weak = false) {
uintptr_t enc = encode_oop(o, skip_live, weak);
assert(decode_oop(enc) == o, "oop encoding should work: " PTR_FORMAT, p2i(o));
assert(decode_cnt_live(enc) == !skip_live, "skip_live encoding should work");
assert(decode_weak(enc) == weak, "weak encoding should work");
assert(decode_not_chunked(enc), "task should not be chunked");
_obj = enc;
}
ShenandoahMarkTask(oop o, int chunk, int pow) {
uintptr_t enc_oop = encode_oop(o);
ShenandoahMarkTask(oop o, bool skip_live, bool weak, int chunk, int pow) {
uintptr_t enc_oop = encode_oop(o, skip_live, weak);
uintptr_t enc_chunk = encode_chunk(chunk);
uintptr_t enc_pow = encode_pow(pow);
uintptr_t enc = enc_oop | enc_chunk | enc_pow;
assert(decode_oop(enc) == o, "oop encoding should work: " PTR_FORMAT, p2i(o));
assert(decode_cnt_live(enc) == !skip_live, "skip_live should be true for chunked tasks");
assert(decode_weak(enc) == weak, "weak encoding should work");
assert(decode_chunk(enc) == chunk, "chunk encoding should work: %d", chunk);
assert(decode_pow(enc) == pow, "pow encoding should work: %d", pow);
assert(!decode_not_chunked(enc), "task should be chunked");
@ -210,6 +235,8 @@ public:
inline int pow() const { return decode_pow(_obj); }
inline bool is_not_chunked() const { return decode_not_chunked(_obj); }
inline bool is_weak() const { return decode_weak(_obj); }
inline bool count_liveness() const { return decode_cnt_live(_obj); }
DEBUG_ONLY(bool is_valid() const;) // Tasks to be pushed/popped must be valid.
@ -232,12 +259,14 @@ private:
static const int pow_max = nth_bit(pow_bits) - 1;
oop _obj;
bool _skip_live;
bool _weak;
int _chunk;
int _pow;
public:
ShenandoahMarkTask(oop o = NULL, int chunk = 0, int pow = 0):
_obj(o), _chunk(chunk), _pow(pow) {
ShenandoahMarkTask(oop o = NULL, bool skip_live = false, bool weak = false, int chunk = 0, int pow = 0):
_obj(o), _skip_live(skip_live), _weak(weak), _chunk(chunk), _pow(pow) {
assert(0 <= chunk && chunk <= chunk_max, "chunk is in range: %d", chunk);
assert(0 <= pow && pow <= pow_max, "pow is in range: %d", pow);
}
@ -248,6 +277,8 @@ public:
inline int chunk() const { return _chunk; }
inline int pow() const { return _pow; }
inline bool is_not_chunked() const { return _chunk == 0; }
inline bool is_weak() const { return _weak; }
inline bool count_liveness() const { return !_skip_live; }
DEBUG_ONLY(bool is_valid() const;) // Tasks to be pushed/popped must be valid.

4
src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp
View File

@ -59,9 +59,9 @@ public:
virtual void doit();
};
class VM_ShenandoahFinalMarkStartEvac: public VM_ShenandoahReferenceOperation {
class VM_ShenandoahFinalMarkStartEvac: public VM_ShenandoahOperation {
public:
VM_ShenandoahFinalMarkStartEvac() : VM_ShenandoahReferenceOperation() {};
VM_ShenandoahFinalMarkStartEvac() : VM_ShenandoahOperation() {};
VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalMarkStartEvac; }
const char* name() const { return "Shenandoah Final Mark and Start Evacuation"; }
virtual void doit();

59
src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
View File

@ -47,6 +47,17 @@
#undef verify_oop
#endif
static bool is_instance_ref_klass(Klass* k) {
return k->is_instance_klass() && InstanceKlass::cast(k)->reference_type() != REF_NONE;
}
class ShenandoahIgnoreReferenceDiscoverer : public ReferenceDiscoverer {
public:
virtual bool discover_reference(oop obj, ReferenceType type) {
return true;
}
};
class ShenandoahVerifyOopClosure : public BasicOopIterateClosure {
private:
const char* _phase;
@ -68,7 +79,12 @@ public:
_map(map),
_ld(ld),
_interior_loc(NULL),
_loc(NULL) { }
_loc(NULL) {
if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references ||
options._verify_marked == ShenandoahVerifier::_verify_marked_disable) {
set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer());
}
}
private:
void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) {
@ -82,7 +98,9 @@ private:
T o = RawAccess<>::oop_load(p);
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
if (is_instance_ref_klass(obj->klass())) {
obj = ShenandoahForwarding::get_forwardee(obj);
}
// Single threaded verification can use faster non-atomic stack and bitmap
// methods.
//
@ -208,6 +226,10 @@ private:
check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj),
"Must be marked in complete bitmap");
break;
case ShenandoahVerifier::_verify_marked_complete_except_references:
check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj),
"Must be marked in complete bitmap, except j.l.r.Reference referents");
break;
default:
assert(false, "Unhandled mark verification");
}
@ -526,19 +548,19 @@ public:
virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) {
size_t processed = 0;
MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map();
HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r);
ShenandoahMarkingContext* ctx = _heap->complete_marking_context();
HeapWord* tams = ctx->top_at_mark_start(r);
// Bitmaps, before TAMS
if (tams > r->bottom()) {
HeapWord* start = r->bottom();
HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
HeapWord* addr = ctx->get_next_marked_addr(start, tams);
while (addr < tams) {
verify_and_follow(addr, stack, cl, &processed);
addr += 1;
if (addr < tams) {
addr = mark_bit_map->get_next_marked_addr(addr, tams);
addr = ctx->get_next_marked_addr(addr, tams);
}
}
}
@ -566,9 +588,10 @@ public:
// Verify everything reachable from that object too, hopefully realizing
// everything was already marked, and never touching further:
cl.verify_oops_from(obj);
(*processed)++;
if (!is_instance_ref_klass(obj->klass())) {
cl.verify_oops_from(obj);
(*processed)++;
}
while (!stack.is_empty()) {
ShenandoahVerifierTask task = stack.pop();
cl.verify_oops_from(task.obj());
@ -718,7 +741,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label,
// version
size_t count_marked = 0;
if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) {
if (ShenandoahVerifyLevel >= 4 && (marked == _verify_marked_complete || marked == _verify_marked_complete_except_references)) {
guarantee(_heap->marking_context()->is_complete(), "Marking context should be complete");
ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options);
_heap->workers()->run_task(&task);
@ -793,11 +816,11 @@ void ShenandoahVerifier::verify_after_concmark() {
verify_at_safepoint(
"After Mark",
_verify_forwarded_none, // no forwarded references
_verify_marked_complete, // bitmaps as precise as we can get
_verify_marked_complete_except_references, // bitmaps as precise as we can get, except dangling j.l.r.Refs
_verify_cset_none, // no references to cset anymore
_verify_liveness_complete, // liveness data must be complete here
_verify_regions_disable, // trash regions not yet recycled
_verify_gcstate_stable, // mark should have stabilized the heap
_verify_gcstate_stable, // mark should have stabilized the heap
_verify_all_weak_roots
);
}
@ -810,12 +833,12 @@ void ShenandoahVerifier::verify_before_evacuation() {
verify_at_safepoint(
"Before Evacuation",
_verify_forwarded_none, // no forwarded references
_verify_marked_complete, // walk over marked objects too
_verify_cset_disable, // non-forwarded references to cset expected
_verify_liveness_complete, // liveness data must be complete here
_verify_regions_disable, // trash regions not yet recycled
_verify_gcstate_stable, // mark should have stabilized the heap
_verify_forwarded_none, // no forwarded references
_verify_marked_complete_except_references, // walk over marked objects too
_verify_cset_disable, // non-forwarded references to cset expected
_verify_liveness_complete, // liveness data must be complete here
_verify_regions_disable, // trash regions not yet recycled
_verify_gcstate_stable, // mark should have stabilized the heap
verify_weak_roots
);
}

6
src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp
View File

@ -65,7 +65,11 @@ public:
_verify_marked_incomplete,
// Objects should be marked in "complete" bitmap.
_verify_marked_complete
_verify_marked_complete,
// Objects should be marked in "complete" bitmap, except j.l.r.Reference referents, which
// may be dangling after marking but before conc-weakrefs-processing.
_verify_marked_complete_except_references
} VerifyMarked;
typedef enum {

22
src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp
View File

@ -32,6 +32,7 @@ uint ShenandoahWorkerPolicy::_prev_par_marking = 0;
uint ShenandoahWorkerPolicy::_prev_conc_marking = 0;
uint ShenandoahWorkerPolicy::_prev_conc_evac = 0;
uint ShenandoahWorkerPolicy::_prev_conc_root_proc = 0;
uint ShenandoahWorkerPolicy::_prev_conc_refs_proc = 0;
uint ShenandoahWorkerPolicy::_prev_fullgc = 0;
uint ShenandoahWorkerPolicy::_prev_degengc = 0;
uint ShenandoahWorkerPolicy::_prev_conc_update_ref = 0;
@ -63,13 +64,23 @@ uint ShenandoahWorkerPolicy::calc_workers_for_final_marking() {
return _prev_par_marking;
}
// Calculate workers for concurrent refs processing
uint ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing() {
uint active_workers = (_prev_conc_refs_proc == 0) ? ConcGCThreads : _prev_conc_refs_proc;
_prev_conc_refs_proc =
WorkerPolicy::calc_active_conc_workers(ConcGCThreads,
active_workers,
Threads::number_of_non_daemon_threads());
return _prev_conc_refs_proc;
}
// Calculate workers for concurrent root processing
uint ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing() {
uint active_workers = (_prev_conc_root_proc == 0) ? ConcGCThreads : _prev_conc_root_proc;
_prev_conc_root_proc =
WorkerPolicy::calc_active_conc_workers(ConcGCThreads,
active_workers,
Threads::number_of_non_daemon_threads());
WorkerPolicy::calc_active_conc_workers(ConcGCThreads,
active_workers,
Threads::number_of_non_daemon_threads());
return _prev_conc_root_proc;
}
@ -123,11 +134,6 @@ uint ShenandoahWorkerPolicy::calc_workers_for_final_update_ref() {
return _prev_par_update_ref;
}
uint ShenandoahWorkerPolicy::calc_workers_for_conc_preclean() {
// Precleaning is single-threaded
return 1;
}
uint ShenandoahWorkerPolicy::calc_workers_for_conc_cleanup() {
uint active_workers = (_prev_conc_cleanup == 0) ? ConcGCThreads : _prev_conc_cleanup;
_prev_conc_cleanup =

7
src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp
View File

@ -32,6 +32,7 @@ private:
static uint _prev_par_marking;
static uint _prev_conc_marking;
static uint _prev_conc_root_proc;
static uint _prev_conc_refs_proc;
static uint _prev_conc_evac;
static uint _prev_fullgc;
static uint _prev_degengc;
@ -53,6 +54,9 @@ public:
// Calculate workers for concurrent root processing
static uint calc_workers_for_conc_root_processing();
// Calculate workers for concurrent refs processing
static uint calc_workers_for_conc_refs_processing();
// Calculate workers for concurrent evacuation (concurrent GC)
static uint calc_workers_for_conc_evac();
@ -68,9 +72,6 @@ public:
// Calculate workers for parallel/final reference update
static uint calc_workers_for_final_update_ref();
// Calculate workers for concurrent precleaning
static uint calc_workers_for_conc_preclean();
// Calculate workers for concurrent cleanup
static uint calc_workers_for_conc_cleanup();

12
src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp
View File

@ -76,13 +76,6 @@
" compact - run GC more frequently and with deeper targets to " \
"free up more memory.") \
\
product(uintx, ShenandoahRefProcFrequency, 5, EXPERIMENTAL, \
"Process process weak (soft, phantom, finalizers) references " \
"every Nth cycle. Normally affects concurrent GC cycles only, " \
"as degenerated and full GCs would try to process references " \
"regardless. Set to zero to disable reference processing " \
"completely.") \
\
product(uintx, ShenandoahUnloadClassesFrequency, 1, EXPERIMENTAL, \
"Unload the classes every Nth cycle. Normally affects concurrent "\
"GC cycles, as degenerated and full GCs would try to unload " \
@ -313,11 +306,6 @@
"Forcefully flush non-empty SATB buffers at this interval. " \
"Time is in milliseconds.") \
\
product(bool, ShenandoahPreclean, true, DIAGNOSTIC, \
"Do concurrent preclean phase before final mark: process " \
"definitely alive references to avoid dealing with them during " \
"pause.") \
\
product(bool, ShenandoahSuspendibleWorkers, false, EXPERIMENTAL, \
"Suspend concurrent GC worker threads at safepoints") \
\

1
test/hotspot/jtreg/gc/TestSoftReferencesBehaviorOnOOME.java
View File

@ -28,6 +28,7 @@ package gc;
* @key randomness
* @summary Tests that all SoftReferences has been cleared at time of OOM.
* @requires vm.gc != "Z"
* @requires vm.gc != "Shenandoah"
* @library /test/lib
* @modules java.base/jdk.internal.misc
* @run main/othervm -Xmx128m gc.TestSoftReferencesBehaviorOnOOME 512 2k