8209420: Track membars for volatile accesses so they can be properly optimized
Reviewed-by: adinn, aph, thartmann
This commit is contained in:
parent
67a1517a4d
commit
f3e518394b
src/hotspot
File diff suppressed because it is too large
Load Diff
@ -119,10 +119,11 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con
|
||||
|
||||
class C2AccessFence: public StackObj {
|
||||
C2Access& _access;
|
||||
Node* _leading_membar;
|
||||
|
||||
public:
|
||||
C2AccessFence(C2Access& access) :
|
||||
_access(access) {
|
||||
_access(access), _leading_membar(NULL) {
|
||||
GraphKit* kit = access.kit();
|
||||
DecoratorSet decorators = access.decorators();
|
||||
|
||||
@ -139,12 +140,12 @@ public:
|
||||
// into actual barriers on most machines, but we still need rest of
|
||||
// compiler to respect ordering.
|
||||
if (is_release) {
|
||||
kit->insert_mem_bar(Op_MemBarRelease);
|
||||
_leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
|
||||
} else if (is_volatile) {
|
||||
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
|
||||
kit->insert_mem_bar(Op_MemBarVolatile);
|
||||
_leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
|
||||
} else {
|
||||
kit->insert_mem_bar(Op_MemBarRelease);
|
||||
_leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
|
||||
}
|
||||
}
|
||||
} else if (is_write) {
|
||||
@ -152,7 +153,7 @@ public:
|
||||
// floating down past the volatile write. Also prevents commoning
|
||||
// another volatile read.
|
||||
if (is_volatile || is_release) {
|
||||
kit->insert_mem_bar(Op_MemBarRelease);
|
||||
_leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
|
||||
}
|
||||
} else {
|
||||
// Memory barrier to prevent normal and 'unsafe' accesses from
|
||||
@ -161,7 +162,7 @@ public:
|
||||
// so there's no problems making a strong assert about mixing users
|
||||
// of safe & unsafe memory.
|
||||
if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
|
||||
kit->insert_mem_bar(Op_MemBarVolatile);
|
||||
_leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
|
||||
}
|
||||
}
|
||||
|
||||
@ -196,20 +197,30 @@ public:
|
||||
|
||||
if (is_atomic) {
|
||||
if (is_acquire || is_volatile) {
|
||||
kit->insert_mem_bar(Op_MemBarAcquire);
|
||||
Node* n = _access.raw_access();
|
||||
Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
|
||||
if (_leading_membar != NULL) {
|
||||
MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
|
||||
}
|
||||
}
|
||||
} else if (is_write) {
|
||||
// If not multiple copy atomic, we do the MemBarVolatile before the load.
|
||||
if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
|
||||
kit->insert_mem_bar(Op_MemBarVolatile); // Use fat membar
|
||||
Node* n = _access.raw_access();
|
||||
Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
|
||||
if (_leading_membar != NULL) {
|
||||
MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (is_volatile || is_acquire) {
|
||||
kit->insert_mem_bar(Op_MemBarAcquire, _access.raw_access());
|
||||
Node* n = _access.raw_access();
|
||||
assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
|
||||
Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
|
||||
mb->as_MemBar()->set_trailing_load();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {
|
||||
|
@ -2767,6 +2767,17 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) {
|
||||
LoadNode::is_immutable_value(n->in(MemNode::Address))),
|
||||
"raw memory operations should have control edge");
|
||||
}
|
||||
if (n->is_MemBar()) {
|
||||
MemBarNode* mb = n->as_MemBar();
|
||||
if (mb->trailing_store() || mb->trailing_load_store()) {
|
||||
assert(mb->leading_membar()->trailing_membar() == mb, "bad membar pair");
|
||||
Node* mem = mb->in(MemBarNode::Precedent);
|
||||
assert((mb->trailing_store() && mem->is_Store() && mem->as_Store()->is_release()) ||
|
||||
(mb->trailing_load_store() && mem->is_LoadStore()), "missing mem op");
|
||||
} else if (mb->leading()) {
|
||||
assert(mb->trailing_membar()->leading_membar() == mb, "bad membar pair");
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// Count FPU ops and common calls, implements item (3)
|
||||
switch( nop ) {
|
||||
|
@ -2521,45 +2521,63 @@ Node* StoreNode::Identity(PhaseGVN* phase) {
|
||||
Node* adr = in(MemNode::Address);
|
||||
Node* val = in(MemNode::ValueIn);
|
||||
|
||||
Node* result = this;
|
||||
|
||||
// Load then Store? Then the Store is useless
|
||||
if (val->is_Load() &&
|
||||
val->in(MemNode::Address)->eqv_uncast(adr) &&
|
||||
val->in(MemNode::Memory )->eqv_uncast(mem) &&
|
||||
val->as_Load()->store_Opcode() == Opcode()) {
|
||||
return mem;
|
||||
result = mem;
|
||||
}
|
||||
|
||||
// Two stores in a row of the same value?
|
||||
if (mem->is_Store() &&
|
||||
if (result == this &&
|
||||
mem->is_Store() &&
|
||||
mem->in(MemNode::Address)->eqv_uncast(adr) &&
|
||||
mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
|
||||
mem->Opcode() == Opcode()) {
|
||||
return mem;
|
||||
result = mem;
|
||||
}
|
||||
|
||||
// Store of zero anywhere into a freshly-allocated object?
|
||||
// Then the store is useless.
|
||||
// (It must already have been captured by the InitializeNode.)
|
||||
if (ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
|
||||
if (result == this &&
|
||||
ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
|
||||
// a newly allocated object is already all-zeroes everywhere
|
||||
if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
|
||||
return mem;
|
||||
result = mem;
|
||||
}
|
||||
|
||||
// the store may also apply to zero-bits in an earlier object
|
||||
Node* prev_mem = find_previous_store(phase);
|
||||
// Steps (a), (b): Walk past independent stores to find an exact match.
|
||||
if (prev_mem != NULL) {
|
||||
Node* prev_val = can_see_stored_value(prev_mem, phase);
|
||||
if (prev_val != NULL && phase->eqv(prev_val, val)) {
|
||||
// prev_val and val might differ by a cast; it would be good
|
||||
// to keep the more informative of the two.
|
||||
return mem;
|
||||
if (result == this) {
|
||||
// the store may also apply to zero-bits in an earlier object
|
||||
Node* prev_mem = find_previous_store(phase);
|
||||
// Steps (a), (b): Walk past independent stores to find an exact match.
|
||||
if (prev_mem != NULL) {
|
||||
Node* prev_val = can_see_stored_value(prev_mem, phase);
|
||||
if (prev_val != NULL && phase->eqv(prev_val, val)) {
|
||||
// prev_val and val might differ by a cast; it would be good
|
||||
// to keep the more informative of the two.
|
||||
result = mem;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return this;
|
||||
if (result != this && phase->is_IterGVN() != NULL) {
|
||||
MemBarNode* trailing = trailing_membar();
|
||||
if (trailing != NULL) {
|
||||
#ifdef ASSERT
|
||||
const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
|
||||
assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
|
||||
#endif
|
||||
PhaseIterGVN* igvn = phase->is_IterGVN();
|
||||
trailing->remove(igvn);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//------------------------------match_edge-------------------------------------
|
||||
@ -2637,6 +2655,33 @@ bool StoreNode::value_never_loaded( PhaseTransform *phase) const {
|
||||
return true;
|
||||
}
|
||||
|
||||
MemBarNode* StoreNode::trailing_membar() const {
|
||||
if (is_release()) {
|
||||
MemBarNode* trailing_mb = NULL;
|
||||
for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
|
||||
Node* u = fast_out(i);
|
||||
if (u->is_MemBar()) {
|
||||
if (u->as_MemBar()->trailing_store()) {
|
||||
assert(u->Opcode() == Op_MemBarVolatile, "");
|
||||
assert(trailing_mb == NULL, "only one");
|
||||
trailing_mb = u->as_MemBar();
|
||||
#ifdef ASSERT
|
||||
Node* leading = u->as_MemBar()->leading_membar();
|
||||
assert(leading->Opcode() == Op_MemBarRelease, "incorrect membar");
|
||||
assert(leading->as_MemBar()->leading_store(), "incorrect membar pair");
|
||||
assert(leading->as_MemBar()->trailing_membar() == u, "incorrect membar pair");
|
||||
#endif
|
||||
} else {
|
||||
assert(u->as_MemBar()->standalone(), "");
|
||||
}
|
||||
}
|
||||
}
|
||||
return trailing_mb;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
//=============================================================================
|
||||
//------------------------------Ideal------------------------------------------
|
||||
// If the store is from an AND mask that leaves the low bits untouched, then
|
||||
@ -2749,6 +2794,30 @@ bool LoadStoreNode::result_not_used() const {
|
||||
return true;
|
||||
}
|
||||
|
||||
MemBarNode* LoadStoreNode::trailing_membar() const {
|
||||
MemBarNode* trailing = NULL;
|
||||
for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
|
||||
Node* u = fast_out(i);
|
||||
if (u->is_MemBar()) {
|
||||
if (u->as_MemBar()->trailing_load_store()) {
|
||||
assert(u->Opcode() == Op_MemBarAcquire, "");
|
||||
assert(trailing == NULL, "only one");
|
||||
trailing = u->as_MemBar();
|
||||
#ifdef ASSERT
|
||||
Node* leading = trailing->leading_membar();
|
||||
assert(support_IRIW_for_not_multiple_copy_atomic_cpu || leading->Opcode() == Op_MemBarRelease, "incorrect membar");
|
||||
assert(leading->as_MemBar()->leading_load_store(), "incorrect membar pair");
|
||||
assert(leading->as_MemBar()->trailing_membar() == trailing, "incorrect membar pair");
|
||||
#endif
|
||||
} else {
|
||||
assert(u->as_MemBar()->standalone(), "wrong barrier kind");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return trailing;
|
||||
}
|
||||
|
||||
uint LoadStoreNode::size_of() const { return sizeof(*this); }
|
||||
|
||||
//=============================================================================
|
||||
@ -2934,7 +3003,10 @@ Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
|
||||
//=============================================================================
|
||||
MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
|
||||
: MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
|
||||
_adr_type(C->get_adr_type(alias_idx))
|
||||
_adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
|
||||
#ifdef ASSERT
|
||||
, _pair_idx(0)
|
||||
#endif
|
||||
{
|
||||
init_class_id(Class_MemBar);
|
||||
Node* top = C->top();
|
||||
@ -2969,6 +3041,21 @@ MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) {
|
||||
}
|
||||
}
|
||||
|
||||
void MemBarNode::remove(PhaseIterGVN *igvn) {
|
||||
if (outcnt() != 2) {
|
||||
return;
|
||||
}
|
||||
if (trailing_store() || trailing_load_store()) {
|
||||
MemBarNode* leading = leading_membar();
|
||||
if (leading != NULL) {
|
||||
assert(leading->trailing_membar() == this, "inconsistent leading/trailing membars");
|
||||
leading->remove(igvn);
|
||||
}
|
||||
}
|
||||
igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
|
||||
igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
|
||||
}
|
||||
|
||||
//------------------------------Ideal------------------------------------------
|
||||
// Return a node which is more "ideal" than the current node. Strip out
|
||||
// control copies
|
||||
@ -3035,8 +3122,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
|
||||
if (eliminate) {
|
||||
// Replace MemBar projections by its inputs.
|
||||
PhaseIterGVN* igvn = phase->is_IterGVN();
|
||||
igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
|
||||
igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
|
||||
remove(igvn);
|
||||
// Must return either the original node (now dead) or a new node
|
||||
// (Do not return a top here, since that would break the uniqueness of top.)
|
||||
return new ConINode(TypeInt::ZERO);
|
||||
@ -3065,6 +3151,98 @@ Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
|
||||
trailing->_kind = TrailingStore;
|
||||
leading->_kind = LeadingStore;
|
||||
#ifdef ASSERT
|
||||
trailing->_pair_idx = leading->_idx;
|
||||
leading->_pair_idx = leading->_idx;
|
||||
#endif
|
||||
}
|
||||
|
||||
void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
|
||||
trailing->_kind = TrailingLoadStore;
|
||||
leading->_kind = LeadingLoadStore;
|
||||
#ifdef ASSERT
|
||||
trailing->_pair_idx = leading->_idx;
|
||||
leading->_pair_idx = leading->_idx;
|
||||
#endif
|
||||
}
|
||||
|
||||
MemBarNode* MemBarNode::trailing_membar() const {
|
||||
Node* trailing = (Node*)this;
|
||||
VectorSet seen(Thread::current()->resource_area());
|
||||
while (!trailing->is_MemBar() || !trailing->as_MemBar()->trailing()) {
|
||||
if (seen.test_set(trailing->_idx)) {
|
||||
// Dying subgraph?
|
||||
return NULL;
|
||||
}
|
||||
for (DUIterator_Fast jmax, j = trailing->fast_outs(jmax); j < jmax; j++) {
|
||||
Node* next = trailing->fast_out(j);
|
||||
if (next != trailing && next->is_CFG()) {
|
||||
trailing = next;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
MemBarNode* mb = trailing->as_MemBar();
|
||||
assert((mb->_kind == TrailingStore && _kind == LeadingStore) ||
|
||||
(mb->_kind == TrailingLoadStore && _kind == LeadingLoadStore), "bad trailing membar");
|
||||
assert(mb->_pair_idx == _pair_idx, "bad trailing membar");
|
||||
return mb;
|
||||
}
|
||||
|
||||
MemBarNode* MemBarNode::leading_membar() const {
|
||||
VectorSet seen(Thread::current()->resource_area());
|
||||
Node* leading = in(0);
|
||||
while (leading != NULL && (!leading->is_MemBar() || !leading->as_MemBar()->leading())) {
|
||||
if (seen.test_set(leading->_idx)) {
|
||||
// Dying subgraph?
|
||||
return NULL;
|
||||
}
|
||||
if (leading->is_Region()) {
|
||||
leading = leading->in(1);
|
||||
} else {
|
||||
leading = leading->in(0);
|
||||
}
|
||||
}
|
||||
#ifdef ASSERT
|
||||
Unique_Node_List wq;
|
||||
wq.push((Node*)this);
|
||||
uint found = 0;
|
||||
for (uint i = 0; i < wq.size(); i++) {
|
||||
Node* n = wq.at(i);
|
||||
if (n->is_Region()) {
|
||||
for (uint j = 1; j < n->req(); j++) {
|
||||
Node* in = n->in(j);
|
||||
if (in != NULL && !in->is_top()) {
|
||||
wq.push(in);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (n->is_MemBar() && n->as_MemBar()->leading()) {
|
||||
assert(n == leading, "consistency check failed");
|
||||
found++;
|
||||
} else {
|
||||
Node* in = n->in(0);
|
||||
if (in != NULL && !in->is_top()) {
|
||||
wq.push(in);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(found == 1 || (found == 0 && leading == NULL), "consistency check failed");
|
||||
#endif
|
||||
if (leading == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
MemBarNode* mb = leading->as_MemBar();
|
||||
assert((mb->_kind == LeadingStore && _kind == TrailingStore) ||
|
||||
(mb->_kind == LeadingLoadStore && _kind == TrailingLoadStore), "bad leading membar");
|
||||
assert(mb->_pair_idx == _pair_idx, "bad leading membar");
|
||||
return mb;
|
||||
}
|
||||
|
||||
//===========================InitializeNode====================================
|
||||
// SUMMARY:
|
||||
// This node acts as a memory barrier on raw memory, after some raw stores.
|
||||
|
@ -607,6 +607,8 @@ public:
|
||||
|
||||
// have all possible loads of the value stored been optimized away?
|
||||
bool value_never_loaded(PhaseTransform *phase) const;
|
||||
|
||||
MemBarNode* trailing_membar() const;
|
||||
};
|
||||
|
||||
//------------------------------StoreBNode-------------------------------------
|
||||
@ -816,6 +818,7 @@ public:
|
||||
virtual const class TypePtr *adr_type() const { return _adr_type; } // returns bottom_type of address
|
||||
|
||||
bool result_not_used() const;
|
||||
MemBarNode* trailing_membar() const;
|
||||
};
|
||||
|
||||
class LoadStoreConditionalNode : public LoadStoreNode {
|
||||
@ -1142,6 +1145,20 @@ class MemBarNode: public MultiNode {
|
||||
// Memory type this node is serializing. Usually either rawptr or bottom.
|
||||
const TypePtr* _adr_type;
|
||||
|
||||
// How is this membar related to a nearby memory access?
|
||||
enum {
|
||||
Standalone,
|
||||
TrailingLoad,
|
||||
TrailingStore,
|
||||
LeadingStore,
|
||||
TrailingLoadStore,
|
||||
LeadingLoadStore
|
||||
} _kind;
|
||||
|
||||
#ifdef ASSERT
|
||||
uint _pair_idx;
|
||||
#endif
|
||||
|
||||
public:
|
||||
enum {
|
||||
Precedent = TypeFunc::Parms // optional edge to force precedence
|
||||
@ -1159,6 +1176,24 @@ public:
|
||||
static MemBarNode* make(Compile* C, int opcode,
|
||||
int alias_idx = Compile::AliasIdxBot,
|
||||
Node* precedent = NULL);
|
||||
|
||||
MemBarNode* trailing_membar() const;
|
||||
MemBarNode* leading_membar() const;
|
||||
|
||||
void set_trailing_load() { _kind = TrailingLoad; }
|
||||
bool trailing_load() const { return _kind == TrailingLoad; }
|
||||
bool trailing_store() const { return _kind == TrailingStore; }
|
||||
bool leading_store() const { return _kind == LeadingStore; }
|
||||
bool trailing_load_store() const { return _kind == TrailingLoadStore; }
|
||||
bool leading_load_store() const { return _kind == LeadingLoadStore; }
|
||||
bool trailing() const { return _kind == TrailingLoad || _kind == TrailingStore || _kind == TrailingLoadStore; }
|
||||
bool leading() const { return _kind == LeadingStore || _kind == LeadingLoadStore; }
|
||||
bool standalone() const { return _kind == Standalone; }
|
||||
|
||||
static void set_store_pair(MemBarNode* leading, MemBarNode* trailing);
|
||||
static void set_load_store_pair(MemBarNode* leading, MemBarNode* trailing);
|
||||
|
||||
void remove(PhaseIterGVN *igvn);
|
||||
};
|
||||
|
||||
// "Acquire" - no following ref can move before (but earlier refs can
|
||||
|
Loading…
x
Reference in New Issue
Block a user