8233401: Shenandoah: Refactor/cleanup Shenandoah load barrier code
Reviewed-by: aph, shade, rkennke
This commit is contained in:
Zhengyu Gu
parent
b2e6cba0b4
commit
476cfd1547
@ -23,6 +23,7 @@
|
||||
|
||||
#include "precompiled.hpp"
|
||||
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
|
||||
#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
|
||||
#include "gc/shenandoah/shenandoahForwarding.hpp"
|
||||
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
|
||||
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
|
||||
@ -346,29 +347,44 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Arguments:
|
||||
//
|
||||
// Inputs:
|
||||
// src: oop location to load from, might be clobbered
|
||||
//
|
||||
// Output:
|
||||
// dst: oop loaded from src location
|
||||
//
|
||||
// Kill:
|
||||
// rscratch1 (scratch reg)
|
||||
//
|
||||
// Alias:
|
||||
// dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
|
||||
//
|
||||
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
|
||||
Register dst, Address src, Register tmp1, Register tmp_thread) {
|
||||
bool on_oop = is_reference_type(type);
|
||||
bool not_in_heap = (decorators & IN_NATIVE) != 0;
|
||||
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
|
||||
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
|
||||
bool on_reference = on_weak || on_phantom;
|
||||
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
|
||||
bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
|
||||
// 1: non-reference load, no additional barrier is needed
|
||||
if (!is_reference_type(type)) {
|
||||
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
|
||||
return;
|
||||
}
|
||||
|
||||
Register result_dst = dst;
|
||||
// 2: load a reference from src location and apply LRB if ShenandoahLoadRefBarrier is set
|
||||
if (ShenandoahLoadRefBarrier) {
|
||||
Register result_dst = dst;
|
||||
|
||||
if (on_oop) {
|
||||
// We want to preserve src
|
||||
// Preserve src location for LRB
|
||||
if (dst == src.base() || dst == src.index()) {
|
||||
dst = rscratch1;
|
||||
}
|
||||
assert_different_registers(dst, src.base(), src.index());
|
||||
}
|
||||
|
||||
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
|
||||
if (on_oop) {
|
||||
if (not_in_heap && !is_traversal_mode) {
|
||||
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
|
||||
|
||||
// Native barrier is for concurrent root processing
|
||||
bool in_native = (decorators & IN_NATIVE) != 0;
|
||||
if (in_native && ShenandoahConcurrentRoots::can_do_concurrent_roots()) {
|
||||
load_reference_barrier_native(masm, dst, src);
|
||||
} else {
|
||||
load_reference_barrier(masm, dst, src);
|
||||
@ -378,8 +394,19 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
|
||||
__ mov(result_dst, dst);
|
||||
dst = result_dst;
|
||||
}
|
||||
} else {
|
||||
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
|
||||
}
|
||||
|
||||
if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
|
||||
// 3: apply keep-alive barrier if ShenandoahKeepAliveBarrier is set
|
||||
if (ShenandoahKeepAliveBarrier) {
|
||||
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
|
||||
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
|
||||
bool on_reference = on_weak || on_phantom;
|
||||
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
|
||||
bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
|
||||
|
||||
if (on_reference && keep_alive) {
|
||||
__ enter();
|
||||
satb_write_barrier_pre(masm /* masm */,
|
||||
noreg /* obj */,
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
|
||||
#include "precompiled.hpp"
|
||||
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
|
||||
#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
|
||||
#include "gc/shenandoah/shenandoahForwarding.hpp"
|
||||
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
|
||||
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
|
||||
@ -445,21 +446,33 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Arguments:
|
||||
//
|
||||
// Inputs:
|
||||
// src: oop location, might be clobbered
|
||||
// tmp1: scratch register, might not be valid.
|
||||
//
|
||||
// Output:
|
||||
// dst: oop loaded from src location
|
||||
//
|
||||
// Kill:
|
||||
// tmp1 (if it is valid)
|
||||
//
|
||||
void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
|
||||
Register dst, Address src, Register tmp1, Register tmp_thread) {
|
||||
bool on_oop = is_reference_type(type);
|
||||
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
|
||||
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
|
||||
bool not_in_heap = (decorators & IN_NATIVE) != 0;
|
||||
bool on_reference = on_weak || on_phantom;
|
||||
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
|
||||
bool keep_alive = ((decorators & AS_NO_KEEPALIVE) == 0) || is_traversal_mode;
|
||||
// 1: non-reference load, no additional barrier is needed
|
||||
if (!is_reference_type(type)) {
|
||||
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
|
||||
return;
|
||||
}
|
||||
|
||||
Register result_dst = dst;
|
||||
bool use_tmp1_for_dst = false;
|
||||
// 2: load a reference from src location and apply LRB if ShenandoahLoadRefBarrier is set
|
||||
if (ShenandoahLoadRefBarrier) {
|
||||
Register result_dst = dst;
|
||||
bool use_tmp1_for_dst = false;
|
||||
|
||||
if (on_oop) {
|
||||
// We want to preserve src
|
||||
// Preserve src location for LRB
|
||||
if (dst == src.base() || dst == src.index()) {
|
||||
// Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
|
||||
if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
|
||||
@ -469,19 +482,20 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
|
||||
dst = rdi;
|
||||
__ push(dst);
|
||||
}
|
||||
assert_different_registers(dst, src.base(), src.index());
|
||||
}
|
||||
assert_different_registers(dst, src.base(), src.index());
|
||||
}
|
||||
|
||||
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
|
||||
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
|
||||
|
||||
if (on_oop) {
|
||||
if (not_in_heap && !is_traversal_mode) {
|
||||
// Native barrier is for concurrent root processing
|
||||
bool in_native = (decorators & IN_NATIVE) != 0;
|
||||
if (in_native && ShenandoahConcurrentRoots::can_do_concurrent_roots()) {
|
||||
load_reference_barrier_native(masm, dst, src);
|
||||
} else {
|
||||
load_reference_barrier(masm, dst, src);
|
||||
}
|
||||
|
||||
// Move loaded oop to final destination
|
||||
if (dst != result_dst) {
|
||||
__ movptr(result_dst, dst);
|
||||
|
||||
@ -491,8 +505,19 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d
|
||||
|
||||
dst = result_dst;
|
||||
}
|
||||
} else {
|
||||
BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
|
||||
}
|
||||
|
||||
if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
|
||||
// 3: apply keep-alive barrier if ShenandoahKeepAliveBarrier is set
|
||||
if (ShenandoahKeepAliveBarrier) {
|
||||
bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
|
||||
bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
|
||||
bool on_reference = on_weak || on_phantom;
|
||||
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
|
||||
bool keep_alive = ((decorators & AS_NO_KEEPALIVE) == 0) || is_traversal_mode;
|
||||
|
||||
if (on_reference && keep_alive) {
|
||||
const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
|
||||
assert_different_registers(dst, tmp1, tmp_thread);
|
||||
NOT_LP64(__ get_thread(thread));
|
||||
|
||||
@ -25,6 +25,7 @@
|
||||
#include "c1/c1_IR.hpp"
|
||||
#include "gc/shared/satbMarkQueue.hpp"
|
||||
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
|
||||
#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
|
||||
#include "gc/shenandoah/shenandoahHeap.hpp"
|
||||
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
|
||||
#include "gc/shenandoah/shenandoahRuntime.hpp"
|
||||
@ -203,46 +204,49 @@ LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_
|
||||
}
|
||||
|
||||
void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
|
||||
// 1: non-reference load, no additional barrier is needed
|
||||
if (!access.is_oop()) {
|
||||
BarrierSetC1::load_at_resolved(access, result);
|
||||
return;
|
||||
}
|
||||
|
||||
LIRGenerator* gen = access.gen();
|
||||
|
||||
DecoratorSet decorators = access.decorators();
|
||||
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
|
||||
|
||||
if ((decorators & IN_NATIVE) != 0 && !is_traversal_mode) {
|
||||
assert(access.is_oop(), "IN_NATIVE access only for oop values");
|
||||
BarrierSetC1::load_at_resolved(access, result);
|
||||
LIR_OprList* args = new LIR_OprList();
|
||||
LIR_Opr addr = access.resolved_addr();
|
||||
addr = ensure_in_register(gen, addr);
|
||||
args->append(result);
|
||||
args->append(addr);
|
||||
BasicTypeList signature;
|
||||
signature.append(T_OBJECT);
|
||||
signature.append(T_ADDRESS);
|
||||
LIR_Opr call_result = gen->call_runtime(&signature, args,
|
||||
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native),
|
||||
objectType, NULL);
|
||||
__ move(call_result, result);
|
||||
} else {
|
||||
if (ShenandoahLoadRefBarrier) {
|
||||
// 2: load a reference from src location and apply LRB if ShenandoahLoadRefBarrier is set
|
||||
if (ShenandoahLoadRefBarrier) {
|
||||
// Native barrier is for concurrent root processing
|
||||
bool in_native = (decorators & IN_NATIVE) != 0;
|
||||
if (in_native && ShenandoahConcurrentRoots::can_do_concurrent_roots()) {
|
||||
BarrierSetC1::load_at_resolved(access, result);
|
||||
LIR_OprList* args = new LIR_OprList();
|
||||
LIR_Opr addr = access.resolved_addr();
|
||||
addr = ensure_in_register(gen, addr);
|
||||
args->append(result);
|
||||
args->append(addr);
|
||||
BasicTypeList signature;
|
||||
signature.append(T_OBJECT);
|
||||
signature.append(T_ADDRESS);
|
||||
LIR_Opr call_result = gen->call_runtime(&signature, args,
|
||||
CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native),
|
||||
objectType, NULL);
|
||||
__ move(call_result, result);
|
||||
} else {
|
||||
LIR_Opr tmp = gen->new_register(T_OBJECT);
|
||||
BarrierSetC1::load_at_resolved(access, tmp);
|
||||
tmp = load_reference_barrier(access.gen(), tmp, access.resolved_addr());
|
||||
tmp = load_reference_barrier(gen, tmp, access.resolved_addr());
|
||||
__ move(tmp, result);
|
||||
} else {
|
||||
BarrierSetC1::load_at_resolved(access, result);
|
||||
}
|
||||
} else {
|
||||
BarrierSetC1::load_at_resolved(access, result);
|
||||
}
|
||||
|
||||
// 3: apply keep-alive barrier if ShenandoahKeepAliveBarrier is set
|
||||
if (ShenandoahKeepAliveBarrier) {
|
||||
bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0;
|
||||
bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
|
||||
bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
|
||||
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
|
||||
bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
|
||||
|
||||
if ((is_weak || is_phantom || is_anonymous) && keep_alive) {
|
||||
@ -252,13 +256,13 @@ void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result)
|
||||
Lcont_anonymous = new LabelObj();
|
||||
generate_referent_check(access, Lcont_anonymous);
|
||||
}
|
||||
pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr /* addr_opr */,
|
||||
pre_barrier(gen, access.access_emit_info(), decorators, LIR_OprFact::illegalOpr /* addr_opr */,
|
||||
result /* pre_val */);
|
||||
if (is_anonymous) {
|
||||
__ branch_destination(Lcont_anonymous->label());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
|
||||
#include "precompiled.hpp"
|
||||
#include "gc/shared/barrierSet.hpp"
|
||||
#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
|
||||
#include "gc/shenandoah/shenandoahForwarding.hpp"
|
||||
#include "gc/shenandoah/shenandoahHeap.hpp"
|
||||
#include "gc/shenandoah/shenandoahHeuristics.hpp"
|
||||
@ -534,66 +535,69 @@ Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue&
|
||||
}
|
||||
|
||||
Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
|
||||
// 1: non-reference load, no additional barrier is needed
|
||||
if (!access.is_oop()) {
|
||||
return BarrierSetC2::load_at_resolved(access, val_type);;
|
||||
}
|
||||
|
||||
Node* load = BarrierSetC2::load_at_resolved(access, val_type);
|
||||
DecoratorSet decorators = access.decorators();
|
||||
|
||||
Node* adr = access.addr().node();
|
||||
Node* obj = access.base();
|
||||
|
||||
bool mismatched = (decorators & C2_MISMATCHED) != 0;
|
||||
bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
|
||||
bool on_heap = (decorators & IN_HEAP) != 0;
|
||||
bool on_weak_ref = (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) != 0;
|
||||
bool is_unordered = (decorators & MO_UNORDERED) != 0;
|
||||
bool need_cpu_mem_bar = !is_unordered || mismatched || !on_heap;
|
||||
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
|
||||
bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
|
||||
bool in_native = (decorators & IN_NATIVE) != 0;
|
||||
|
||||
Node* top = Compile::current()->top();
|
||||
|
||||
Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
|
||||
Node* load = BarrierSetC2::load_at_resolved(access, val_type);
|
||||
|
||||
if (access.is_oop()) {
|
||||
if (ShenandoahLoadRefBarrier) {
|
||||
load = new ShenandoahLoadReferenceBarrierNode(NULL, load, in_native && !is_traversal_mode);
|
||||
if (access.is_parse_access()) {
|
||||
load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
|
||||
} else {
|
||||
load = static_cast<C2OptAccess &>(access).gvn().transform(load);
|
||||
}
|
||||
// 2: apply LRB if ShenandoahLoadRefBarrier is set
|
||||
if (ShenandoahLoadRefBarrier) {
|
||||
// Native barrier is for concurrent root processing
|
||||
bool use_native_barrier = in_native && ShenandoahConcurrentRoots::can_do_concurrent_roots();
|
||||
load = new ShenandoahLoadReferenceBarrierNode(NULL, load, use_native_barrier);
|
||||
if (access.is_parse_access()) {
|
||||
load = static_cast<C2ParseAccess &>(access).kit()->gvn().transform(load);
|
||||
} else {
|
||||
load = static_cast<C2OptAccess &>(access).gvn().transform(load);
|
||||
}
|
||||
}
|
||||
|
||||
// If we are reading the value of the referent field of a Reference
|
||||
// object (either by using Unsafe directly or through reflection)
|
||||
// then, if SATB is enabled, we need to record the referent in an
|
||||
// SATB log buffer using the pre-barrier mechanism.
|
||||
// Also we need to add memory barrier to prevent commoning reads
|
||||
// from this field across safepoint since GC can change its value.
|
||||
bool need_read_barrier = ShenandoahKeepAliveBarrier &&
|
||||
(on_weak_ref || (unknown && offset != top && obj != top));
|
||||
// 3: apply keep-alive barrier if ShenandoahKeepAliveBarrier is set
|
||||
if (ShenandoahKeepAliveBarrier) {
|
||||
Node* top = Compile::current()->top();
|
||||
Node* adr = access.addr().node();
|
||||
Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top;
|
||||
Node* obj = access.base();
|
||||
|
||||
if (!access.is_oop() || !need_read_barrier) {
|
||||
return load;
|
||||
}
|
||||
bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0;
|
||||
bool on_weak_ref = (decorators & (ON_WEAK_OOP_REF | ON_PHANTOM_OOP_REF)) != 0;
|
||||
bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
|
||||
bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
|
||||
|
||||
assert(access.is_parse_access(), "entry not supported at optimization time");
|
||||
C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
|
||||
GraphKit* kit = parse_access.kit();
|
||||
// If we are reading the value of the referent field of a Reference
|
||||
// object (either by using Unsafe directly or through reflection)
|
||||
// then, if SATB is enabled, we need to record the referent in an
|
||||
// SATB log buffer using the pre-barrier mechanism.
|
||||
// Also we need to add memory barrier to prevent commoning reads
|
||||
// from this field across safepoint since GC can change its value.
|
||||
if (!on_weak_ref || (unknown && (offset == top || obj == top)) || !keep_alive) {
|
||||
return load;
|
||||
}
|
||||
|
||||
if (on_weak_ref && keep_alive) {
|
||||
// Use the pre-barrier to record the value in the referent field
|
||||
satb_write_barrier_pre(kit, false /* do_load */,
|
||||
NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
|
||||
load /* pre_val */, T_OBJECT);
|
||||
// Add memory barrier to prevent commoning reads from this field
|
||||
// across safepoint since GC can change its value.
|
||||
kit->insert_mem_bar(Op_MemBarCPUOrder);
|
||||
} else if (unknown) {
|
||||
// We do not require a mem bar inside pre_barrier if need_mem_bar
|
||||
// is set: the barriers would be emitted by us.
|
||||
insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
|
||||
assert(access.is_parse_access(), "entry not supported at optimization time");
|
||||
C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
|
||||
GraphKit* kit = parse_access.kit();
|
||||
bool mismatched = (decorators & C2_MISMATCHED) != 0;
|
||||
bool is_unordered = (decorators & MO_UNORDERED) != 0;
|
||||
bool need_cpu_mem_bar = !is_unordered || mismatched || in_native;
|
||||
|
||||
if (on_weak_ref) {
|
||||
// Use the pre-barrier to record the value in the referent field
|
||||
satb_write_barrier_pre(kit, false /* do_load */,
|
||||
NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,
|
||||
load /* pre_val */, T_OBJECT);
|
||||
// Add memory barrier to prevent commoning reads from this field
|
||||
// across safepoint since GC can change its value.
|
||||
kit->insert_mem_bar(Op_MemBarCPUOrder);
|
||||
} else if (unknown) {
|
||||
// We do not require a mem bar inside pre_barrier if need_mem_bar
|
||||
// is set: the barriers would be emitted by us.
|
||||
insert_pre_barrier(kit, obj, offset, load, !need_cpu_mem_bar);
|
||||
}
|
||||
}
|
||||
|
||||
return load;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user