8224675: Late GC barrier insertion for ZGC
Reviewed-by: roland, eosterlund, pliden
This commit is contained in:
Nils Eliasson
parent
b34b2d993c
commit
75e9d0a290
src/hotspot
cpu/x86/gc/z
share
adlc
compiler
gc
shared/c2
shenandoah/c2
z
opto
classes.hppcompile.cppcompile.hppidealGraphPrinter.cppidealGraphPrinter.hpplcm.cpploopnode.cpploopnode.hpploopopts.cppmemnode.cppmemnode.hppnode.cppnode.hppphaseX.cppphasetype.hppvectornode.cpp
runtime
utilities
@ -21,6 +21,12 @@
|
||||
// questions.
|
||||
//
|
||||
|
||||
source_hpp %{
|
||||
|
||||
#include "gc/z/c2/zBarrierSetC2.hpp"
|
||||
|
||||
%}
|
||||
|
||||
source %{
|
||||
|
||||
#include "gc/z/zBarrierSetAssembler.hpp"
|
||||
@ -45,7 +51,7 @@ instruct zLoadBarrierSlowRegXmmAndYmm(rRegP dst, memory src, rFlagsReg cr,
|
||||
rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
|
||||
|
||||
match(Set dst (LoadBarrierSlowReg src));
|
||||
predicate(UseAVX <= 2);
|
||||
predicate((UseAVX <= 2) && !n->as_LoadBarrierSlowReg()->is_weak());
|
||||
|
||||
effect(DEF dst, KILL cr,
|
||||
KILL x0, KILL x1, KILL x2, KILL x3,
|
||||
@ -74,7 +80,7 @@ instruct zLoadBarrierSlowRegZmm(rRegP dst, memory src, rFlagsReg cr,
|
||||
rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
|
||||
|
||||
match(Set dst (LoadBarrierSlowReg src));
|
||||
predicate(UseAVX == 3);
|
||||
predicate((UseAVX == 3) && !n->as_LoadBarrierSlowReg()->is_weak());
|
||||
|
||||
effect(DEF dst, KILL cr,
|
||||
KILL x0, KILL x1, KILL x2, KILL x3,
|
||||
@ -102,8 +108,8 @@ instruct zLoadBarrierWeakSlowRegXmmAndYmm(rRegP dst, memory src, rFlagsReg cr,
|
||||
rxmm8 x8, rxmm9 x9, rxmm10 x10, rxmm11 x11,
|
||||
rxmm12 x12, rxmm13 x13, rxmm14 x14, rxmm15 x15) %{
|
||||
|
||||
match(Set dst (LoadBarrierWeakSlowReg src));
|
||||
predicate(UseAVX <= 2);
|
||||
match(Set dst (LoadBarrierSlowReg src));
|
||||
predicate((UseAVX <= 2) && n->as_LoadBarrierSlowReg()->is_weak());
|
||||
|
||||
effect(DEF dst, KILL cr,
|
||||
KILL x0, KILL x1, KILL x2, KILL x3,
|
||||
@ -131,8 +137,8 @@ instruct zLoadBarrierWeakSlowRegZmm(rRegP dst, memory src, rFlagsReg cr,
|
||||
rxmm24 x24, rxmm25 x25, rxmm26 x26, rxmm27 x27,
|
||||
rxmm28 x28, rxmm29 x29, rxmm30 x30, rxmm31 x31) %{
|
||||
|
||||
match(Set dst (LoadBarrierWeakSlowReg src));
|
||||
predicate(UseAVX == 3);
|
||||
match(Set dst (LoadBarrierSlowReg src));
|
||||
predicate((UseAVX == 3) && n->as_LoadBarrierSlowReg()->is_weak());
|
||||
|
||||
effect(DEF dst, KILL cr,
|
||||
KILL x0, KILL x1, KILL x2, KILL x3,
|
||||
@ -152,3 +158,58 @@ instruct zLoadBarrierWeakSlowRegZmm(rRegP dst, memory src, rFlagsReg cr,
|
||||
|
||||
ins_pipe(pipe_slow);
|
||||
%}
|
||||
|
||||
// Specialized versions of compareAndExchangeP that adds a keepalive that is consumed
|
||||
// but doesn't affect output.
|
||||
|
||||
instruct z_compareAndExchangeP(
|
||||
memory mem_ptr,
|
||||
rax_RegP oldval, rRegP newval, rRegP keepalive,
|
||||
rFlagsReg cr) %{
|
||||
predicate(VM_Version::supports_cx8());
|
||||
match(Set oldval (ZCompareAndExchangeP (Binary mem_ptr keepalive) (Binary oldval newval)));
|
||||
effect(KILL cr);
|
||||
|
||||
format %{ "cmpxchgq $mem_ptr,$newval\t# "
|
||||
"If rax == $mem_ptr then store $newval into $mem_ptr\n\t" %}
|
||||
opcode(0x0F, 0xB1);
|
||||
ins_encode(lock_prefix,
|
||||
REX_reg_mem_wide(newval, mem_ptr),
|
||||
OpcP, OpcS,
|
||||
reg_mem(newval, mem_ptr) // lock cmpxchg
|
||||
);
|
||||
ins_pipe( pipe_cmpxchg );
|
||||
%}
|
||||
|
||||
instruct z_compareAndSwapP(rRegI res,
|
||||
memory mem_ptr,
|
||||
rax_RegP oldval, rRegP newval, rRegP keepalive,
|
||||
rFlagsReg cr) %{
|
||||
predicate(VM_Version::supports_cx8());
|
||||
match(Set res (ZCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval)));
|
||||
match(Set res (ZWeakCompareAndSwapP (Binary mem_ptr keepalive) (Binary oldval newval)));
|
||||
effect(KILL cr, KILL oldval);
|
||||
|
||||
format %{ "cmpxchgq $mem_ptr,$newval\t# "
|
||||
"If rax == $mem_ptr then store $newval into $mem_ptr\n\t"
|
||||
"sete $res\n\t"
|
||||
"movzbl $res, $res" %}
|
||||
opcode(0x0F, 0xB1);
|
||||
ins_encode(lock_prefix,
|
||||
REX_reg_mem_wide(newval, mem_ptr),
|
||||
OpcP, OpcS,
|
||||
reg_mem(newval, mem_ptr),
|
||||
REX_breg(res), Opcode(0x0F), Opcode(0x94), reg(res), // sete
|
||||
REX_reg_breg(res, res), // movzbl
|
||||
Opcode(0xF), Opcode(0xB6), reg_reg(res, res));
|
||||
ins_pipe( pipe_cmpxchg );
|
||||
%}
|
||||
|
||||
instruct z_xchgP( memory mem, rRegP newval, rRegP keepalive) %{
|
||||
match(Set newval (ZGetAndSetP mem (Binary newval keepalive)));
|
||||
format %{ "XCHGQ $newval,[$mem]" %}
|
||||
ins_encode %{
|
||||
__ xchgq($newval$$Register, $mem$$Address);
|
||||
%}
|
||||
ins_pipe( pipe_cmpxchg );
|
||||
%}
|
||||
|
||||
@ -757,7 +757,6 @@ int InstructForm::memory_operand(FormDict &globals) const {
|
||||
return NO_MEMORY_OPERAND;
|
||||
}
|
||||
|
||||
|
||||
// This instruction captures the machine-independent bottom_type
|
||||
// Expected use is for pointer vs oop determination for LoadP
|
||||
bool InstructForm::captures_bottom_type(FormDict &globals) const {
|
||||
@ -775,8 +774,9 @@ bool InstructForm::captures_bottom_type(FormDict &globals) const {
|
||||
!strcmp(_matrule->_rChild->_opType,"GetAndSetP") ||
|
||||
!strcmp(_matrule->_rChild->_opType,"GetAndSetN") ||
|
||||
#if INCLUDE_ZGC
|
||||
!strcmp(_matrule->_rChild->_opType,"ZGetAndSetP") ||
|
||||
!strcmp(_matrule->_rChild->_opType,"ZCompareAndExchangeP") ||
|
||||
!strcmp(_matrule->_rChild->_opType,"LoadBarrierSlowReg") ||
|
||||
!strcmp(_matrule->_rChild->_opType,"LoadBarrierWeakSlowReg") ||
|
||||
#endif
|
||||
#if INCLUDE_SHENANDOAHGC
|
||||
!strcmp(_matrule->_rChild->_opType,"ShenandoahCompareAndExchangeP") ||
|
||||
@ -3506,12 +3506,16 @@ int MatchNode::needs_ideal_memory_edge(FormDict &globals) const {
|
||||
"CompareAndSwapB", "CompareAndSwapS", "CompareAndSwapI", "CompareAndSwapL", "CompareAndSwapP", "CompareAndSwapN",
|
||||
"WeakCompareAndSwapB", "WeakCompareAndSwapS", "WeakCompareAndSwapI", "WeakCompareAndSwapL", "WeakCompareAndSwapP", "WeakCompareAndSwapN",
|
||||
"CompareAndExchangeB", "CompareAndExchangeS", "CompareAndExchangeI", "CompareAndExchangeL", "CompareAndExchangeP", "CompareAndExchangeN",
|
||||
#if INCLUDE_SHENANDOAHGC
|
||||
"ShenandoahCompareAndSwapN", "ShenandoahCompareAndSwapP", "ShenandoahWeakCompareAndSwapP", "ShenandoahWeakCompareAndSwapN", "ShenandoahCompareAndExchangeP", "ShenandoahCompareAndExchangeN",
|
||||
#endif
|
||||
"StoreCM",
|
||||
"ClearArray",
|
||||
"GetAndSetB", "GetAndSetS", "GetAndAddI", "GetAndSetI", "GetAndSetP",
|
||||
"GetAndAddB", "GetAndAddS", "GetAndAddL", "GetAndSetL", "GetAndSetN",
|
||||
"LoadBarrierSlowReg", "LoadBarrierWeakSlowReg"
|
||||
#if INCLUDE_ZGC
|
||||
"LoadBarrierSlowReg", "ZGetAndSetP", "ZCompareAndSwapP", "ZCompareAndExchangeP", "ZWeakCompareAndSwapP",
|
||||
#endif
|
||||
"ClearArray"
|
||||
};
|
||||
int cnt = sizeof(needs_ideal_memory_list)/sizeof(char*);
|
||||
if( strcmp(_opType,"PrefetchAllocation")==0 )
|
||||
|
||||
@ -67,7 +67,7 @@ NOT_PRODUCT(cflags(TraceOptoOutput, bool, TraceOptoOutput, TraceOptoOutput))
|
||||
cflags(CloneMapDebug, bool, false, CloneMapDebug) \
|
||||
cflags(IGVPrintLevel, intx, PrintIdealGraphLevel, IGVPrintLevel) \
|
||||
cflags(MaxNodeLimit, intx, MaxNodeLimit, MaxNodeLimit) \
|
||||
ZGC_ONLY(cflags(ZOptimizeLoadBarriers, bool, ZOptimizeLoadBarriers, ZOptimizeLoadBarriers))
|
||||
ZGC_ONLY(cflags(ZTraceLoadBarriers, bool, false, ZTraceLoadBarriers))
|
||||
#else
|
||||
#define compilerdirectives_c2_flags(cflags)
|
||||
#endif
|
||||
|
||||
@ -259,6 +259,7 @@ public:
|
||||
Optimization,
|
||||
Expansion
|
||||
};
|
||||
|
||||
virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const { return false; }
|
||||
virtual void clone_barrier_at_expansion(ArrayCopyNode* ac, Node* call, PhaseIterGVN& igvn) const;
|
||||
|
||||
@ -273,7 +274,6 @@ public:
|
||||
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { }
|
||||
virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {}
|
||||
virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {}
|
||||
virtual void add_users_to_worklist(Unique_Node_List* worklist) const {}
|
||||
|
||||
// Allow barrier sets to have shared state that is preserved across a compilation unit.
|
||||
// This could for example comprise macro nodes to be expanded during macro expansion.
|
||||
@ -286,17 +286,21 @@ public:
|
||||
virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return false; }
|
||||
|
||||
virtual bool has_special_unique_user(const Node* node) const { return false; }
|
||||
virtual bool needs_anti_dependence_check(const Node* node) const { return true; }
|
||||
|
||||
virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const { }
|
||||
|
||||
enum CompilePhase {
|
||||
BeforeOptimize, /* post_parse = true */
|
||||
BeforeExpand, /* post_parse = false */
|
||||
BeforeOptimize,
|
||||
BeforeLateInsertion,
|
||||
BeforeMacroExpand,
|
||||
BeforeCodeGen
|
||||
};
|
||||
virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const {}
|
||||
|
||||
virtual bool flatten_gc_alias_type(const TypePtr*& adr_type) const { return false; }
|
||||
#ifdef ASSERT
|
||||
virtual bool verify_gc_alias_type(const TypePtr* adr_type, int offset) const { return false; }
|
||||
virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const {}
|
||||
#endif
|
||||
|
||||
virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode) const { return false; }
|
||||
@ -310,8 +314,8 @@ public:
|
||||
virtual bool matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const { return false; };
|
||||
virtual bool matcher_is_store_load_barrier(Node* x, uint xop) const { return false; }
|
||||
|
||||
virtual void igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const {}
|
||||
virtual void ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const {}
|
||||
virtual void igvn_add_users_to_worklist(PhaseIterGVN* igvn, Node* use) const { }
|
||||
virtual void ccp_analyze(PhaseCCP* ccp, Unique_Node_List& worklist, Node* use) const { }
|
||||
|
||||
virtual Node* split_if_pre(PhaseIdealLoop* phase, Node* n) const { return NULL; }
|
||||
virtual bool build_loop_late_post(PhaseIdealLoop* phase, Node* n) const { return false; }
|
||||
|
||||
@ -912,8 +912,6 @@ void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &use
|
||||
}
|
||||
}
|
||||
|
||||
void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {}
|
||||
|
||||
void* ShenandoahBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
|
||||
return new(comp_arena) ShenandoahBarrierSetC2State(comp_arena);
|
||||
}
|
||||
@ -928,7 +926,7 @@ bool ShenandoahBarrierSetC2::expand_macro_nodes(PhaseMacroExpand* macro) const {
|
||||
|
||||
#ifdef ASSERT
|
||||
void ShenandoahBarrierSetC2::verify_gc_barriers(Compile* compile, CompilePhase phase) const {
|
||||
if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeExpand) {
|
||||
if (ShenandoahVerifyOptoBarriers && phase == BarrierSetC2::BeforeMacroExpand) {
|
||||
ShenandoahBarrierC2Support::verify(Compile::current()->root());
|
||||
} else if (phase == BarrierSetC2::BeforeCodeGen) {
|
||||
// Verify G1 pre-barriers
|
||||
|
||||
@ -126,7 +126,6 @@ public:
|
||||
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
|
||||
virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const;
|
||||
virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const;
|
||||
virtual void add_users_to_worklist(Unique_Node_List* worklist) const;
|
||||
|
||||
// Allow barrier sets to have shared state that is preserved across a compilation unit.
|
||||
// This could for example comprise macro nodes to be expanded during macro expansion.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -29,15 +29,33 @@
|
||||
#include "opto/node.hpp"
|
||||
#include "utilities/growableArray.hpp"
|
||||
|
||||
class ZCompareAndSwapPNode : public CompareAndSwapPNode {
|
||||
public:
|
||||
ZCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : CompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { }
|
||||
virtual int Opcode() const;
|
||||
};
|
||||
|
||||
class ZWeakCompareAndSwapPNode : public WeakCompareAndSwapPNode {
|
||||
public:
|
||||
ZWeakCompareAndSwapPNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, MemNode::MemOrd mem_ord) : WeakCompareAndSwapPNode(c, mem, adr, val, ex, mem_ord) { }
|
||||
virtual int Opcode() const;
|
||||
};
|
||||
|
||||
class ZCompareAndExchangePNode : public CompareAndExchangePNode {
|
||||
public:
|
||||
ZCompareAndExchangePNode(Node* c, Node *mem, Node *adr, Node *val, Node *ex, const TypePtr* at, const Type* t, MemNode::MemOrd mem_ord) : CompareAndExchangePNode(c, mem, adr, val, ex, at, t, mem_ord) { }
|
||||
virtual int Opcode() const;
|
||||
};
|
||||
|
||||
class ZGetAndSetPNode : public GetAndSetPNode {
|
||||
public:
|
||||
ZGetAndSetPNode(Node* c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* t) : GetAndSetPNode(c, mem, adr, val, at, t) { }
|
||||
virtual int Opcode() const;
|
||||
};
|
||||
|
||||
class LoadBarrierNode : public MultiNode {
|
||||
private:
|
||||
bool _weak; // On strong or weak oop reference
|
||||
bool _writeback; // Controls if the barrier writes the healed oop back to memory
|
||||
// A swap on a memory location must never write back the healed oop
|
||||
bool _oop_reload_allowed; // Controls if the barrier are allowed to reload the oop from memory
|
||||
// before healing, otherwise both the oop and the address must be
|
||||
// passed to the barrier from the oop
|
||||
|
||||
static bool is_dominator(PhaseIdealLoop* phase, bool linear_only, Node *d, Node *n);
|
||||
void push_dominated_barriers(PhaseIterGVN* igvn) const;
|
||||
|
||||
@ -57,9 +75,7 @@ public:
|
||||
Node* mem,
|
||||
Node* val,
|
||||
Node* adr,
|
||||
bool weak,
|
||||
bool writeback,
|
||||
bool oop_reload_allowed);
|
||||
bool weak);
|
||||
|
||||
virtual int Opcode() const;
|
||||
virtual uint size_of() const;
|
||||
@ -86,17 +102,11 @@ public:
|
||||
bool is_weak() const {
|
||||
return _weak;
|
||||
}
|
||||
|
||||
bool is_writeback() const {
|
||||
return _writeback;
|
||||
}
|
||||
|
||||
bool oop_reload_allowed() const {
|
||||
return _oop_reload_allowed;
|
||||
}
|
||||
};
|
||||
|
||||
class LoadBarrierSlowRegNode : public LoadPNode {
|
||||
private:
|
||||
bool _is_weak;
|
||||
public:
|
||||
LoadBarrierSlowRegNode(Node *c,
|
||||
Node *mem,
|
||||
@ -104,8 +114,9 @@ public:
|
||||
const TypePtr *at,
|
||||
const TypePtr* t,
|
||||
MemOrd mo,
|
||||
bool weak = false,
|
||||
ControlDependency control_dependency = DependsOnlyOnTest) :
|
||||
LoadPNode(c, mem, adr, at, t, mo, control_dependency) {
|
||||
LoadPNode(c, mem, adr, at, t, mo, control_dependency), _is_weak(weak) {
|
||||
init_class_id(Class_LoadBarrierSlowReg);
|
||||
}
|
||||
|
||||
@ -118,30 +129,8 @@ public:
|
||||
}
|
||||
|
||||
virtual int Opcode() const;
|
||||
};
|
||||
|
||||
class LoadBarrierWeakSlowRegNode : public LoadPNode {
|
||||
public:
|
||||
LoadBarrierWeakSlowRegNode(Node *c,
|
||||
Node *mem,
|
||||
Node *adr,
|
||||
const TypePtr *at,
|
||||
const TypePtr* t,
|
||||
MemOrd mo,
|
||||
ControlDependency control_dependency = DependsOnlyOnTest) :
|
||||
LoadPNode(c, mem, adr, at, t, mo, control_dependency) {
|
||||
init_class_id(Class_LoadBarrierWeakSlowReg);
|
||||
}
|
||||
|
||||
virtual const char * name() {
|
||||
return "LoadBarrierWeakSlowRegNode";
|
||||
}
|
||||
|
||||
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
virtual int Opcode() const;
|
||||
bool is_weak() { return _is_weak; }
|
||||
};
|
||||
|
||||
class ZBarrierSetC2State : public ResourceObj {
|
||||
@ -157,15 +146,17 @@ public:
|
||||
LoadBarrierNode* load_barrier_node(int idx) const;
|
||||
};
|
||||
|
||||
enum BarrierInfo {
|
||||
NoBarrier = 0,
|
||||
RequireBarrier = 1,
|
||||
WeakBarrier = 3, // Inclusive with RequireBarrier
|
||||
ExpandedBarrier = 4
|
||||
};
|
||||
|
||||
class ZBarrierSetC2 : public BarrierSetC2 {
|
||||
private:
|
||||
ZBarrierSetC2State* state() const;
|
||||
Node* make_cas_loadbarrier(C2AtomicParseAccess& access) const;
|
||||
Node* make_cmpx_loadbarrier(C2AtomicParseAccess& access) const;
|
||||
void expand_loadbarrier_basic(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const;
|
||||
void expand_loadbarrier_node(PhaseMacroExpand* phase, LoadBarrierNode* barrier) const;
|
||||
void expand_loadbarrier_optimized(PhaseMacroExpand* phase, LoadBarrierNode *barrier) const;
|
||||
const TypeFunc* load_barrier_Type() const;
|
||||
|
||||
#ifdef ASSERT
|
||||
void verify_gc_barriers(bool post_parse) const;
|
||||
@ -186,41 +177,42 @@ protected:
|
||||
const Type* val_type) const;
|
||||
|
||||
public:
|
||||
Node* load_barrier(GraphKit* kit,
|
||||
Node* val,
|
||||
Node* adr,
|
||||
bool weak = false,
|
||||
bool writeback = true,
|
||||
bool oop_reload_allowed = true) const;
|
||||
|
||||
virtual void* create_barrier_state(Arena* comp_arena) const;
|
||||
|
||||
virtual bool has_load_barriers() const { return true; }
|
||||
virtual bool is_gc_barrier_node(Node* node) const;
|
||||
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { }
|
||||
virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const;
|
||||
virtual void add_users_to_worklist(Unique_Node_List* worklist) const;
|
||||
virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const;
|
||||
virtual Node* step_over_gc_barrier(Node* c) const;
|
||||
|
||||
virtual void register_potential_barrier_node(Node* node) const;
|
||||
virtual void unregister_potential_barrier_node(Node* node) const;
|
||||
virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { }
|
||||
virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const;
|
||||
virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const;
|
||||
|
||||
virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const;
|
||||
virtual Node* step_over_gc_barrier(Node* c) const;
|
||||
// If the BarrierSetC2 state has kept barrier nodes in its compilation unit state to be
|
||||
// expanded later, then now is the time to do so.
|
||||
|
||||
virtual bool expand_barriers(Compile* C, PhaseIterGVN& igvn) const;
|
||||
|
||||
static void find_dominating_barriers(PhaseIterGVN& igvn);
|
||||
static void loop_optimize_gc_barrier(PhaseIdealLoop* phase, Node* node, bool last_round);
|
||||
|
||||
virtual bool final_graph_reshaping(Compile* compile, Node* n, uint opcode) const;
|
||||
|
||||
virtual bool matcher_find_shared_visit(Matcher* matcher, Matcher::MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) const;
|
||||
virtual bool matcher_find_shared_post_visit(Matcher* matcher, Node* n, uint opcode) const;
|
||||
virtual bool needs_anti_dependence_check(const Node* node) const;
|
||||
|
||||
#ifdef ASSERT
|
||||
virtual void verify_gc_barriers(Compile* compile, CompilePhase phase) const;
|
||||
#endif
|
||||
|
||||
virtual bool escape_add_to_con_graph(ConnectionGraph* conn_graph, PhaseGVN* gvn, Unique_Node_List* delayed_worklist, Node* n, uint opcode) const;
|
||||
virtual bool escape_add_final_edges(ConnectionGraph* conn_graph, PhaseGVN* gvn, Node* n, uint opcode) const;
|
||||
// Load barrier insertion and expansion external
|
||||
virtual void barrier_insertion_phase(Compile* C, PhaseIterGVN &igvn) const;
|
||||
virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const;
|
||||
virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return (mode == LoopOptsZBarrierInsertion); }
|
||||
|
||||
private:
|
||||
// Load barrier insertion and expansion internal
|
||||
void insert_barriers_on_unsafe(PhaseIdealLoop* phase) const;
|
||||
void clean_catch_blocks(PhaseIdealLoop* phase) const;
|
||||
void insert_load_barriers(PhaseIdealLoop* phase) const;
|
||||
LoadNode* insert_one_loadbarrier(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl) const;
|
||||
void insert_one_loadbarrier_inner(PhaseIdealLoop* phase, LoadNode* load, Node* ctrl, VectorSet visited) const;
|
||||
};
|
||||
|
||||
#endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP
|
||||
|
||||
@ -328,46 +328,9 @@ void ZHeap::mark_flush_and_free(Thread* thread) {
|
||||
_mark.flush_and_free(thread);
|
||||
}
|
||||
|
||||
class ZFixupPartialLoadsClosure : public ZRootsIteratorClosure {
|
||||
public:
|
||||
virtual void do_oop(oop* p) {
|
||||
ZBarrier::mark_barrier_on_root_oop_field(p);
|
||||
}
|
||||
|
||||
virtual void do_oop(narrowOop* p) {
|
||||
ShouldNotReachHere();
|
||||
}
|
||||
};
|
||||
|
||||
class ZFixupPartialLoadsTask : public ZTask {
|
||||
private:
|
||||
ZThreadRootsIterator _thread_roots;
|
||||
|
||||
public:
|
||||
ZFixupPartialLoadsTask() :
|
||||
ZTask("ZFixupPartialLoadsTask"),
|
||||
_thread_roots() {}
|
||||
|
||||
virtual void work() {
|
||||
ZFixupPartialLoadsClosure cl;
|
||||
_thread_roots.oops_do(&cl);
|
||||
}
|
||||
};
|
||||
|
||||
void ZHeap::fixup_partial_loads() {
|
||||
ZFixupPartialLoadsTask task;
|
||||
_workers.run_parallel(&task);
|
||||
}
|
||||
|
||||
bool ZHeap::mark_end() {
|
||||
assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
|
||||
|
||||
// C2 can generate code where a safepoint poll is inserted
|
||||
// between a load and the associated load barrier. To handle
|
||||
// this case we need to rescan the thread stack here to make
|
||||
// sure such oops are marked.
|
||||
fixup_partial_loads();
|
||||
|
||||
// Try end marking
|
||||
if (!_mark.end()) {
|
||||
// Marking not completed, continue concurrent mark
|
||||
|
||||
@ -82,9 +82,6 @@
|
||||
diagnostic(bool, ZVerifyForwarding, false, \
|
||||
"Verify forwarding tables") \
|
||||
\
|
||||
diagnostic(bool, ZOptimizeLoadBarriers, true, \
|
||||
"Apply load barrier optimizations") \
|
||||
\
|
||||
develop(bool, ZVerifyLoadBarriers, false, \
|
||||
"Verify that reference loads are followed by barriers")
|
||||
|
||||
|
||||
@ -196,7 +196,10 @@ macro(LoadS)
|
||||
#endif
|
||||
zgcmacro(LoadBarrier)
|
||||
zgcmacro(LoadBarrierSlowReg)
|
||||
zgcmacro(LoadBarrierWeakSlowReg)
|
||||
zgcmacro(ZCompareAndSwapP)
|
||||
zgcmacro(ZWeakCompareAndSwapP)
|
||||
zgcmacro(ZCompareAndExchangeP)
|
||||
zgcmacro(ZGetAndSetP)
|
||||
macro(Lock)
|
||||
macro(Loop)
|
||||
macro(LoopLimit)
|
||||
|
||||
@ -2211,8 +2211,8 @@ void Compile::Optimize() {
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef ASSERT
|
||||
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
|
||||
#ifdef ASSERT
|
||||
bs->verify_gc_barriers(this, BarrierSetC2::BeforeOptimize);
|
||||
#endif
|
||||
|
||||
@ -2371,7 +2371,6 @@ void Compile::Optimize() {
|
||||
igvn = ccp;
|
||||
igvn.optimize();
|
||||
}
|
||||
|
||||
print_method(PHASE_ITER_GVN2, 2);
|
||||
|
||||
if (failing()) return;
|
||||
@ -2382,12 +2381,6 @@ void Compile::Optimize() {
|
||||
return;
|
||||
}
|
||||
|
||||
#if INCLUDE_ZGC
|
||||
if (UseZGC) {
|
||||
ZBarrierSetC2::find_dominating_barriers(igvn);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (failing()) return;
|
||||
|
||||
// Ensure that major progress is now clear
|
||||
@ -2407,28 +2400,33 @@ void Compile::Optimize() {
|
||||
}
|
||||
|
||||
#ifdef ASSERT
|
||||
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
|
||||
bs->verify_gc_barriers(this, BarrierSetC2::BeforeExpand);
|
||||
bs->verify_gc_barriers(this, BarrierSetC2::BeforeLateInsertion);
|
||||
#endif
|
||||
|
||||
bs->barrier_insertion_phase(C, igvn);
|
||||
if (failing()) return;
|
||||
|
||||
#ifdef ASSERT
|
||||
bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
|
||||
#endif
|
||||
|
||||
{
|
||||
TracePhase tp("macroExpand", &timers[_t_macroExpand]);
|
||||
PhaseMacroExpand mex(igvn);
|
||||
print_method(PHASE_BEFORE_MACRO_EXPANSION, 2);
|
||||
if (mex.expand_macro_nodes()) {
|
||||
assert(failing(), "must bail out w/ explicit message");
|
||||
return;
|
||||
}
|
||||
print_method(PHASE_MACRO_EXPANSION, 2);
|
||||
}
|
||||
|
||||
{
|
||||
TracePhase tp("barrierExpand", &timers[_t_barrierExpand]);
|
||||
print_method(PHASE_BEFORE_BARRIER_EXPAND, 2);
|
||||
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
|
||||
if (bs->expand_barriers(this, igvn)) {
|
||||
assert(failing(), "must bail out w/ explicit message");
|
||||
return;
|
||||
}
|
||||
print_method(PHASE_BARRIER_EXPANSION, 2);
|
||||
}
|
||||
|
||||
if (opaque4_count() > 0) {
|
||||
@ -2824,7 +2822,7 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) {
|
||||
MemBarNode* mb = n->as_MemBar();
|
||||
if (mb->trailing_store() || mb->trailing_load_store()) {
|
||||
assert(mb->leading_membar()->trailing_membar() == mb, "bad membar pair");
|
||||
Node* mem = mb->in(MemBarNode::Precedent);
|
||||
Node* mem = BarrierSet::barrier_set()->barrier_set_c2()->step_over_gc_barrier(mb->in(MemBarNode::Precedent));
|
||||
assert((mb->trailing_store() && mem->is_Store() && mem->as_Store()->is_release()) ||
|
||||
(mb->trailing_load_store() && mem->is_LoadStore()), "missing mem op");
|
||||
} else if (mb->leading()) {
|
||||
|
||||
@ -52,6 +52,7 @@ class C2Compiler;
|
||||
class CallGenerator;
|
||||
class CloneMap;
|
||||
class ConnectionGraph;
|
||||
class IdealGraphPrinter;
|
||||
class InlineTree;
|
||||
class Int_Array;
|
||||
class LoadBarrierNode;
|
||||
@ -95,9 +96,9 @@ enum LoopOptsMode {
|
||||
LoopOptsNone,
|
||||
LoopOptsShenandoahExpand,
|
||||
LoopOptsShenandoahPostExpand,
|
||||
LoopOptsZBarrierInsertion,
|
||||
LoopOptsSkipSplitIf,
|
||||
LoopOptsVerify,
|
||||
LoopOptsLastRound
|
||||
LoopOptsVerify
|
||||
};
|
||||
|
||||
typedef unsigned int node_idx_t;
|
||||
@ -658,6 +659,7 @@ class Compile : public Phase {
|
||||
void set_do_cleanup(bool z) { _do_cleanup = z; }
|
||||
int do_cleanup() const { return _do_cleanup; }
|
||||
void set_major_progress() { _major_progress++; }
|
||||
void restore_major_progress(int progress) { _major_progress += progress; }
|
||||
void clear_major_progress() { _major_progress = 0; }
|
||||
int max_inline_size() const { return _max_inline_size; }
|
||||
void set_freq_inline_size(int n) { _freq_inline_size = n; }
|
||||
@ -747,7 +749,15 @@ class Compile : public Phase {
|
||||
C->_latest_stage_start_counter.stamp();
|
||||
}
|
||||
|
||||
void print_method(CompilerPhaseType cpt, int level = 1) {
|
||||
bool should_print(int level = 1) {
|
||||
#ifndef PRODUCT
|
||||
return (_printer && _printer->should_print(level));
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
void print_method(CompilerPhaseType cpt, int level = 1, int idx = 0) {
|
||||
EventCompilerPhase event;
|
||||
if (event.should_commit()) {
|
||||
event.set_starttime(C->_latest_stage_start_counter);
|
||||
@ -757,10 +767,15 @@ class Compile : public Phase {
|
||||
event.commit();
|
||||
}
|
||||
|
||||
|
||||
#ifndef PRODUCT
|
||||
if (_printer && _printer->should_print(level)) {
|
||||
_printer->print_method(CompilerPhaseTypeHelper::to_string(cpt), level);
|
||||
if (should_print(level)) {
|
||||
char output[1024];
|
||||
if (idx != 0) {
|
||||
sprintf(output, "%s:%d", CompilerPhaseTypeHelper::to_string(cpt), idx);
|
||||
} else {
|
||||
sprintf(output, "%s", CompilerPhaseTypeHelper::to_string(cpt));
|
||||
}
|
||||
_printer->print_method(output, level);
|
||||
}
|
||||
#endif
|
||||
C->_latest_stage_start_counter.stamp();
|
||||
|
||||
@ -350,14 +350,6 @@ void IdealGraphPrinter::end_method() {
|
||||
_xml->flush();
|
||||
}
|
||||
|
||||
// Print indent
|
||||
void IdealGraphPrinter::print_indent() {
|
||||
tty->print_cr("printing indent %d", _depth);
|
||||
for (int i = 0; i < _depth; i++) {
|
||||
_xml->print("%s", INDENT);
|
||||
}
|
||||
}
|
||||
|
||||
bool IdealGraphPrinter::traverse_outs() {
|
||||
return _traverse_outs;
|
||||
}
|
||||
@ -663,14 +655,16 @@ void IdealGraphPrinter::walk_nodes(Node *start, bool edges, VectorSet* temp_set)
|
||||
}
|
||||
}
|
||||
|
||||
void IdealGraphPrinter::print_method(const char *name, int level, bool clear_nodes) {
|
||||
print(name, (Node *)C->root(), level, clear_nodes);
|
||||
void IdealGraphPrinter::print_method(const char *name, int level) {
|
||||
if (should_print(level)) {
|
||||
print(name, (Node *) C->root());
|
||||
}
|
||||
}
|
||||
|
||||
// Print current ideal graph
|
||||
void IdealGraphPrinter::print(const char *name, Node *node, int level, bool clear_nodes) {
|
||||
void IdealGraphPrinter::print(const char *name, Node *node) {
|
||||
|
||||
if (!_current_method || !_should_send_method || !should_print(level)) return;
|
||||
if (!_current_method || !_should_send_method) return;
|
||||
|
||||
// Warning, unsafe cast?
|
||||
_chaitin = (PhaseChaitin *)C->regalloc();
|
||||
|
||||
@ -81,11 +81,7 @@ class IdealGraphPrinter : public CHeapObj<mtCompiler> {
|
||||
static const char *METHOD_SHORT_NAME_PROPERTY;
|
||||
static const char *ASSEMBLY_ELEMENT;
|
||||
|
||||
elapsedTimer _walk_time;
|
||||
elapsedTimer _output_time;
|
||||
elapsedTimer _build_blocks_time;
|
||||
|
||||
static int _file_count;
|
||||
static int _file_count;
|
||||
networkStream *_stream;
|
||||
xmlStream *_xml;
|
||||
outputStream *_output;
|
||||
@ -97,10 +93,6 @@ class IdealGraphPrinter : public CHeapObj<mtCompiler> {
|
||||
bool _traverse_outs;
|
||||
Compile *C;
|
||||
|
||||
static void pre_node(Node* node, void *env);
|
||||
static void post_node(Node* node, void *env);
|
||||
|
||||
void print_indent();
|
||||
void print_method(ciMethod *method, int bci, InlineTree *tree);
|
||||
void print_inline_tree(InlineTree *tree);
|
||||
void visit_node(Node *n, bool edges, VectorSet* temp_set);
|
||||
@ -116,7 +108,6 @@ class IdealGraphPrinter : public CHeapObj<mtCompiler> {
|
||||
void tail(const char *name);
|
||||
void head(const char *name);
|
||||
void text(const char *s);
|
||||
intptr_t get_node_id(Node *n);
|
||||
IdealGraphPrinter();
|
||||
~IdealGraphPrinter();
|
||||
|
||||
@ -130,9 +121,8 @@ class IdealGraphPrinter : public CHeapObj<mtCompiler> {
|
||||
void print_inlining();
|
||||
void begin_method();
|
||||
void end_method();
|
||||
void print_method(const char *name, int level=1, bool clear_nodes = false);
|
||||
void print(const char *name, Node *root, int level=1, bool clear_nodes = false);
|
||||
void print_xml(const char *name);
|
||||
void print_method(const char *name, int level = 0);
|
||||
void print(const char *name, Node *root);
|
||||
bool should_print(int level);
|
||||
void set_compile(Compile* compile) {C = compile; }
|
||||
};
|
||||
|
||||
@ -171,7 +171,6 @@ void PhaseCFG::implicit_null_check(Block* block, Node *proj, Node *val, int allo
|
||||
case Op_LoadL:
|
||||
case Op_LoadP:
|
||||
case Op_LoadBarrierSlowReg:
|
||||
case Op_LoadBarrierWeakSlowReg:
|
||||
case Op_LoadN:
|
||||
case Op_LoadS:
|
||||
case Op_LoadKlass:
|
||||
|
||||
@ -978,7 +978,7 @@ void LoopNode::verify_strip_mined(int expect_skeleton) const {
|
||||
wq.push(u);
|
||||
bool found_sfpt = false;
|
||||
for (uint next = 0; next < wq.size() && !found_sfpt; next++) {
|
||||
Node *n = wq.at(next);
|
||||
Node* n = wq.at(next);
|
||||
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && !found_sfpt; i++) {
|
||||
Node* u = n->fast_out(i);
|
||||
if (u == sfpt) {
|
||||
@ -992,6 +992,19 @@ void LoopNode::verify_strip_mined(int expect_skeleton) const {
|
||||
assert(found_sfpt, "no node in loop that's not input to safepoint");
|
||||
}
|
||||
}
|
||||
|
||||
if (UseZGC && !inner_out->in(0)->is_CountedLoopEnd()) {
|
||||
// In some very special cases there can be a load that has no other uses than the
|
||||
// counted loop safepoint. Then its loadbarrier will be placed between the inner
|
||||
// loop exit and the safepoint. This is very rare
|
||||
|
||||
Node* ifnode = inner_out->in(1)->in(0);
|
||||
// Region->IfTrue->If == Region->Iffalse->If
|
||||
if (ifnode == inner_out->in(2)->in(0)) {
|
||||
inner_out = ifnode->in(0);
|
||||
}
|
||||
}
|
||||
|
||||
CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd();
|
||||
assert(cle == inner->loopexit_or_null(), "mismatch");
|
||||
bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0;
|
||||
@ -2761,7 +2774,7 @@ bool PhaseIdealLoop::process_expensive_nodes() {
|
||||
// Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to
|
||||
// its corresponding LoopNode. If 'optimize' is true, do some loop cleanups.
|
||||
void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
|
||||
bool do_split_ifs = (mode == LoopOptsDefault || mode == LoopOptsLastRound);
|
||||
bool do_split_ifs = (mode == LoopOptsDefault);
|
||||
bool skip_loop_opts = (mode == LoopOptsNone);
|
||||
|
||||
int old_progress = C->major_progress();
|
||||
@ -2921,9 +2934,7 @@ void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
|
||||
build_loop_late( visited, worklist, nstack );
|
||||
|
||||
if (_verify_only) {
|
||||
// restore major progress flag
|
||||
for (int i = 0; i < old_progress; i++)
|
||||
C->set_major_progress();
|
||||
C->restore_major_progress(old_progress);
|
||||
assert(C->unique() == unique, "verification mode made Nodes? ? ?");
|
||||
assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything");
|
||||
return;
|
||||
@ -2967,9 +2978,7 @@ void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
|
||||
|
||||
if (skip_loop_opts) {
|
||||
// restore major progress flag
|
||||
for (int i = 0; i < old_progress; i++) {
|
||||
C->set_major_progress();
|
||||
}
|
||||
C->restore_major_progress(old_progress);
|
||||
|
||||
// Cleanup any modified bits
|
||||
_igvn.optimize();
|
||||
@ -3018,11 +3027,8 @@ void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) {
|
||||
// that require basic-block info (like cloning through Phi's)
|
||||
if( SplitIfBlocks && do_split_ifs ) {
|
||||
visited.Clear();
|
||||
split_if_with_blocks( visited, nstack, mode == LoopOptsLastRound );
|
||||
split_if_with_blocks( visited, nstack);
|
||||
NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); );
|
||||
if (mode == LoopOptsLastRound) {
|
||||
C->set_major_progress();
|
||||
}
|
||||
}
|
||||
|
||||
if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) {
|
||||
@ -3157,8 +3163,7 @@ void PhaseIdealLoop::verify() const {
|
||||
_ltree_root->verify_tree(loop_verify._ltree_root, NULL);
|
||||
// Reset major-progress. It was cleared by creating a verify version of
|
||||
// PhaseIdealLoop.
|
||||
for( int i=0; i<old_progress; i++ )
|
||||
C->set_major_progress();
|
||||
C->restore_major_progress(old_progress);
|
||||
}
|
||||
|
||||
//------------------------------verify_compare---------------------------------
|
||||
@ -4288,7 +4293,6 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) {
|
||||
case Op_LoadS:
|
||||
case Op_LoadP:
|
||||
case Op_LoadBarrierSlowReg:
|
||||
case Op_LoadBarrierWeakSlowReg:
|
||||
case Op_LoadN:
|
||||
case Op_LoadRange:
|
||||
case Op_LoadD_unaligned:
|
||||
|
||||
@ -824,6 +824,7 @@ public:
|
||||
// pull such a subsumed block out of the array, we write back the final
|
||||
// correct block.
|
||||
Node *get_ctrl( Node *i ) {
|
||||
|
||||
assert(has_node(i), "");
|
||||
Node *n = get_ctrl_no_update(i);
|
||||
_nodes.map( i->_idx, (Node*)((intptr_t)n + 1) );
|
||||
@ -1306,9 +1307,9 @@ public:
|
||||
|
||||
// Check for aggressive application of 'split-if' optimization,
|
||||
// using basic block level info.
|
||||
void split_if_with_blocks ( VectorSet &visited, Node_Stack &nstack, bool last_round );
|
||||
void split_if_with_blocks ( VectorSet &visited, Node_Stack &nstack);
|
||||
Node *split_if_with_blocks_pre ( Node *n );
|
||||
void split_if_with_blocks_post( Node *n, bool last_round );
|
||||
void split_if_with_blocks_post( Node *n );
|
||||
Node *has_local_phi_input( Node *n );
|
||||
// Mark an IfNode as being dominated by a prior test,
|
||||
// without actually altering the CFG (and hence IDOM info).
|
||||
|
||||
@ -1195,11 +1195,11 @@ bool PhaseIdealLoop::can_split_if(Node* n_ctrl) {
|
||||
// Do the real work in a non-recursive function. CFG hackery wants to be
|
||||
// in the post-order, so it can dirty the I-DOM info and not use the dirtied
|
||||
// info.
|
||||
void PhaseIdealLoop::split_if_with_blocks_post(Node *n, bool last_round) {
|
||||
void PhaseIdealLoop::split_if_with_blocks_post(Node *n) {
|
||||
|
||||
// Cloning Cmp through Phi's involves the split-if transform.
|
||||
// FastLock is not used by an If
|
||||
if (n->is_Cmp() && !n->is_FastLock() && !last_round) {
|
||||
if (n->is_Cmp() && !n->is_FastLock()) {
|
||||
Node *n_ctrl = get_ctrl(n);
|
||||
// Determine if the Node has inputs from some local Phi.
|
||||
// Returns the block to clone thru.
|
||||
@ -1451,18 +1451,12 @@ void PhaseIdealLoop::split_if_with_blocks_post(Node *n, bool last_round) {
|
||||
get_loop(get_ctrl(n)) == get_loop(get_ctrl(n->in(1))) ) {
|
||||
_igvn.replace_node( n, n->in(1) );
|
||||
}
|
||||
|
||||
#if INCLUDE_ZGC
|
||||
if (UseZGC) {
|
||||
ZBarrierSetC2::loop_optimize_gc_barrier(this, n, last_round);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
//------------------------------split_if_with_blocks---------------------------
|
||||
// Check for aggressive application of 'split-if' optimization,
|
||||
// using basic block level info.
|
||||
void PhaseIdealLoop::split_if_with_blocks(VectorSet &visited, Node_Stack &nstack, bool last_round) {
|
||||
void PhaseIdealLoop::split_if_with_blocks(VectorSet &visited, Node_Stack &nstack) {
|
||||
Node* root = C->root();
|
||||
visited.set(root->_idx); // first, mark root as visited
|
||||
// Do pre-visit work for root
|
||||
@ -1488,7 +1482,7 @@ void PhaseIdealLoop::split_if_with_blocks(VectorSet &visited, Node_Stack &nstack
|
||||
// All of n's children have been processed, complete post-processing.
|
||||
if (cnt != 0 && !n->is_Con()) {
|
||||
assert(has_node(n), "no dead nodes");
|
||||
split_if_with_blocks_post(n, last_round);
|
||||
split_if_with_blocks_post(n);
|
||||
}
|
||||
if (must_throttle_split_if()) {
|
||||
nstack.clear();
|
||||
|
||||
@ -908,14 +908,6 @@ static bool skip_through_membars(Compile::AliasType* atp, const TypeInstPtr* tp,
|
||||
// a load node that reads from the source array so we may be able to
|
||||
// optimize out the ArrayCopy node later.
|
||||
Node* LoadNode::can_see_arraycopy_value(Node* st, PhaseGVN* phase) const {
|
||||
#if INCLUDE_ZGC
|
||||
if (UseZGC) {
|
||||
if (bottom_type()->make_oopptr() != NULL) {
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
Node* ld_adr = in(MemNode::Address);
|
||||
intptr_t ld_off = 0;
|
||||
AllocateNode* ld_alloc = AllocateNode::Ideal_allocation(ld_adr, phase, ld_off);
|
||||
@ -2811,7 +2803,8 @@ const Type* SCMemProjNode::Value(PhaseGVN* phase) const
|
||||
LoadStoreNode::LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required )
|
||||
: Node(required),
|
||||
_type(rt),
|
||||
_adr_type(at)
|
||||
_adr_type(at),
|
||||
_has_barrier(false)
|
||||
{
|
||||
init_req(MemNode::Control, c );
|
||||
init_req(MemNode::Memory , mem);
|
||||
@ -3105,16 +3098,6 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if INCLUDE_ZGC
|
||||
if (UseZGC) {
|
||||
if (req() == (Precedent+1) && in(MemBarNode::Precedent)->in(0) != NULL && in(MemBarNode::Precedent)->in(0)->is_LoadBarrier()) {
|
||||
Node* load_node = in(MemBarNode::Precedent)->in(0)->in(LoadBarrierNode::Oop);
|
||||
set_req(MemBarNode::Precedent, load_node);
|
||||
return this;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
bool progress = false;
|
||||
// Eliminate volatile MemBars for scalar replaced objects.
|
||||
if (can_reshape && req() == (Precedent+1)) {
|
||||
|
||||
@ -164,6 +164,7 @@ public:
|
||||
Pinned,
|
||||
DependsOnlyOnTest
|
||||
};
|
||||
|
||||
private:
|
||||
// LoadNode::hash() doesn't take the _control_dependency field
|
||||
// into account: If the graph already has a non-pinned LoadNode and
|
||||
@ -182,6 +183,8 @@ private:
|
||||
// this field.
|
||||
const MemOrd _mo;
|
||||
|
||||
uint _barrier; // Bit field with barrier information
|
||||
|
||||
protected:
|
||||
virtual bool cmp(const Node &n) const;
|
||||
virtual uint size_of() const; // Size is bigger
|
||||
@ -193,7 +196,7 @@ protected:
|
||||
public:
|
||||
|
||||
LoadNode(Node *c, Node *mem, Node *adr, const TypePtr* at, const Type *rt, MemOrd mo, ControlDependency control_dependency)
|
||||
: MemNode(c,mem,adr,at), _control_dependency(control_dependency), _mo(mo), _type(rt) {
|
||||
: MemNode(c,mem,adr,at), _control_dependency(control_dependency), _mo(mo), _barrier(0), _type(rt) {
|
||||
init_class_id(Class_Load);
|
||||
}
|
||||
inline bool is_unordered() const { return !is_acquire(); }
|
||||
@ -262,6 +265,10 @@ public:
|
||||
Node* convert_to_unsigned_load(PhaseGVN& gvn);
|
||||
Node* convert_to_signed_load(PhaseGVN& gvn);
|
||||
|
||||
void copy_barrier_info(const Node* src) { _barrier = src->as_Load()->_barrier; }
|
||||
uint barrier_data() { return _barrier; }
|
||||
void set_barrier_data(uint barrier_data) { _barrier |= barrier_data; }
|
||||
|
||||
#ifndef PRODUCT
|
||||
virtual void dump_spec(outputStream *st) const;
|
||||
#endif
|
||||
@ -810,6 +817,7 @@ class LoadStoreNode : public Node {
|
||||
private:
|
||||
const Type* const _type; // What kind of value is loaded?
|
||||
const TypePtr* _adr_type; // What kind of memory is being addressed?
|
||||
bool _has_barrier;
|
||||
virtual uint size_of() const; // Size is bigger
|
||||
public:
|
||||
LoadStoreNode( Node *c, Node *mem, Node *adr, Node *val, const TypePtr* at, const Type* rt, uint required );
|
||||
@ -822,6 +830,8 @@ public:
|
||||
|
||||
bool result_not_used() const;
|
||||
MemBarNode* trailing_membar() const;
|
||||
void set_has_barrier() { _has_barrier = true; };
|
||||
bool has_barrier() const { return _has_barrier; };
|
||||
};
|
||||
|
||||
class LoadStoreConditionalNode : public LoadStoreNode {
|
||||
|
||||
@ -546,6 +546,9 @@ Node *Node::clone() const {
|
||||
if (n->is_SafePoint()) {
|
||||
n->as_SafePoint()->clone_replaced_nodes();
|
||||
}
|
||||
if (n->is_Load()) {
|
||||
n->as_Load()->copy_barrier_info(this);
|
||||
}
|
||||
return n; // Return the clone
|
||||
}
|
||||
|
||||
@ -564,7 +567,6 @@ void Node::setup_is_top() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//------------------------------~Node------------------------------------------
|
||||
// Fancy destructor; eagerly attempt to reclaim Node numberings and storage
|
||||
void Node::destruct() {
|
||||
@ -1454,13 +1456,16 @@ bool Node::rematerialize() const {
|
||||
//------------------------------needs_anti_dependence_check---------------------
|
||||
// Nodes which use memory without consuming it, hence need antidependences.
|
||||
bool Node::needs_anti_dependence_check() const {
|
||||
if( req() < 2 || (_flags & Flag_needs_anti_dependence_check) == 0 )
|
||||
if (req() < 2 || (_flags & Flag_needs_anti_dependence_check) == 0) {
|
||||
return false;
|
||||
else
|
||||
return in(1)->bottom_type()->has_memory();
|
||||
}
|
||||
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
|
||||
if (!bs->needs_anti_dependence_check(this)) {
|
||||
return false;
|
||||
}
|
||||
return in(1)->bottom_type()->has_memory();
|
||||
}
|
||||
|
||||
|
||||
// Get an integer constant from a ConNode (or CastIINode).
|
||||
// Return a default value if there is no apparent constant here.
|
||||
const TypeInt* Node::find_int_type() const {
|
||||
|
||||
@ -83,8 +83,8 @@ class JumpProjNode;
|
||||
class LoadNode;
|
||||
class LoadBarrierNode;
|
||||
class LoadBarrierSlowRegNode;
|
||||
class LoadBarrierWeakSlowRegNode;
|
||||
class LoadStoreNode;
|
||||
class LoadStoreConditionalNode;
|
||||
class LockNode;
|
||||
class LoopNode;
|
||||
class MachBranchNode;
|
||||
@ -688,8 +688,7 @@ public:
|
||||
DEFINE_CLASS_ID(Mem, Node, 4)
|
||||
DEFINE_CLASS_ID(Load, Mem, 0)
|
||||
DEFINE_CLASS_ID(LoadVector, Load, 0)
|
||||
DEFINE_CLASS_ID(LoadBarrierSlowReg, Load, 1)
|
||||
DEFINE_CLASS_ID(LoadBarrierWeakSlowReg, Load, 2)
|
||||
DEFINE_CLASS_ID(LoadBarrierSlowReg, Load, 1)
|
||||
DEFINE_CLASS_ID(Store, Mem, 1)
|
||||
DEFINE_CLASS_ID(StoreVector, Store, 0)
|
||||
DEFINE_CLASS_ID(LoadStore, Mem, 2)
|
||||
@ -830,9 +829,9 @@ public:
|
||||
DEFINE_CLASS_QUERY(JumpProj)
|
||||
DEFINE_CLASS_QUERY(Load)
|
||||
DEFINE_CLASS_QUERY(LoadStore)
|
||||
DEFINE_CLASS_QUERY(LoadStoreConditional)
|
||||
DEFINE_CLASS_QUERY(LoadBarrier)
|
||||
DEFINE_CLASS_QUERY(LoadBarrierSlowReg)
|
||||
DEFINE_CLASS_QUERY(LoadBarrierWeakSlowReg)
|
||||
DEFINE_CLASS_QUERY(Lock)
|
||||
DEFINE_CLASS_QUERY(Loop)
|
||||
DEFINE_CLASS_QUERY(Mach)
|
||||
|
||||
@ -1003,9 +1003,6 @@ PhaseIterGVN::PhaseIterGVN( PhaseGVN *gvn ) : PhaseGVN(gvn),
|
||||
n->is_Mem() )
|
||||
add_users_to_worklist(n);
|
||||
}
|
||||
|
||||
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
|
||||
bs->add_users_to_worklist(&_worklist);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -52,8 +52,11 @@ enum CompilerPhaseType {
|
||||
PHASE_MATCHING,
|
||||
PHASE_INCREMENTAL_INLINE,
|
||||
PHASE_INCREMENTAL_BOXING_INLINE,
|
||||
PHASE_BEFORE_BARRIER_EXPAND,
|
||||
PHASE_BEFORE_MACRO_EXPANSION,
|
||||
PHASE_CALL_CATCH_CLEANUP,
|
||||
PHASE_INSERT_BARRIER,
|
||||
PHASE_MACRO_EXPANSION,
|
||||
PHASE_BARRIER_EXPANSION,
|
||||
PHASE_ADD_UNSAFE_BARRIER,
|
||||
PHASE_END,
|
||||
PHASE_FAILURE,
|
||||
|
||||
@ -90,8 +93,11 @@ class CompilerPhaseTypeHelper {
|
||||
case PHASE_MATCHING: return "After matching";
|
||||
case PHASE_INCREMENTAL_INLINE: return "Incremental Inline";
|
||||
case PHASE_INCREMENTAL_BOXING_INLINE: return "Incremental Boxing Inline";
|
||||
case PHASE_BEFORE_BARRIER_EXPAND: return "Before Barrier Expand";
|
||||
case PHASE_BEFORE_MACRO_EXPANSION: return "Before macro expansion";
|
||||
case PHASE_CALL_CATCH_CLEANUP: return "Call catch cleanup";
|
||||
case PHASE_INSERT_BARRIER: return "Insert barrier";
|
||||
case PHASE_MACRO_EXPANSION: return "Macro expand";
|
||||
case PHASE_BARRIER_EXPANSION: return "Barrier expand";
|
||||
case PHASE_ADD_UNSAFE_BARRIER: return "Add barrier to unsafe op";
|
||||
case PHASE_END: return "End";
|
||||
case PHASE_FAILURE: return "Failure";
|
||||
default:
|
||||
|
||||
@ -298,7 +298,6 @@ void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
|
||||
case Op_LoadF: case Op_LoadD:
|
||||
case Op_LoadP: case Op_LoadN:
|
||||
case Op_LoadBarrierSlowReg:
|
||||
case Op_LoadBarrierWeakSlowReg:
|
||||
*start = 0;
|
||||
*end = 0; // no vector operands
|
||||
break;
|
||||
|
||||
@ -133,16 +133,12 @@ StackValue* StackValue::create_stack_value(const frame* fr, const RegisterMap* r
|
||||
}
|
||||
#endif
|
||||
// Deoptimization must make sure all oops have passed load barriers
|
||||
#if INCLUDE_ZGC
|
||||
if (UseZGC) {
|
||||
val = ZBarrier::load_barrier_on_oop_field_preloaded((oop*)value_addr, val);
|
||||
}
|
||||
#endif
|
||||
#if INCLUDE_SHENANDOAHGC
|
||||
if (UseShenandoahGC) {
|
||||
val = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(val);
|
||||
}
|
||||
#endif
|
||||
assert(oopDesc::is_oop_or_null(val, false), "bad oop found");
|
||||
Handle h(Thread::current(), val); // Wrap a handle around the oop
|
||||
return new StackValue(h);
|
||||
}
|
||||
|
||||
@ -152,6 +152,12 @@ class GenericGrowableArray : public ResourceObj {
|
||||
template<class E> class GrowableArrayIterator;
|
||||
template<class E, class UnaryPredicate> class GrowableArrayFilterIterator;
|
||||
|
||||
template<class E>
|
||||
class CompareClosure : public Closure {
|
||||
public:
|
||||
virtual int do_compare(const E&, const E&) = 0;
|
||||
};
|
||||
|
||||
template<class E> class GrowableArray : public GenericGrowableArray {
|
||||
friend class VMStructs;
|
||||
|
||||
@ -443,6 +449,37 @@ template<class E> class GrowableArray : public GenericGrowableArray {
|
||||
}
|
||||
return min;
|
||||
}
|
||||
|
||||
E insert_sorted(CompareClosure<E>* cc, const E& key) {
|
||||
bool found;
|
||||
int location = find_sorted(cc, key, found);
|
||||
if (!found) {
|
||||
insert_before(location, key);
|
||||
}
|
||||
return at(location);
|
||||
}
|
||||
|
||||
template<typename K>
|
||||
int find_sorted(CompareClosure<E>* cc, const K& key, bool& found) {
|
||||
found = false;
|
||||
int min = 0;
|
||||
int max = length() - 1;
|
||||
|
||||
while (max >= min) {
|
||||
int mid = (int)(((uint)max + min) / 2);
|
||||
E value = at(mid);
|
||||
int diff = cc->do_compare(key, value);
|
||||
if (diff > 0) {
|
||||
min = mid + 1;
|
||||
} else if (diff < 0) {
|
||||
max = mid - 1;
|
||||
} else {
|
||||
found = true;
|
||||
return mid;
|
||||
}
|
||||
}
|
||||
return min;
|
||||
}
|
||||
};
|
||||
|
||||
// Global GrowableArray methods (one instance in the library per each 'E' type).
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user