8209420: Track membars for volatile accesses so they can be properly optimized

Reviewed-by: adinn, aph, thartmann
This commit is contained in:
Roland Westrelin 2018-08-14 16:54:47 +02:00
parent 67a1517a4d
commit f3e518394b
5 changed files with 304 additions and 1496 deletions
src/hotspot

File diff suppressed because it is too large Load Diff

@ -119,10 +119,11 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con
class C2AccessFence: public StackObj {
C2Access& _access;
Node* _leading_membar;
public:
C2AccessFence(C2Access& access) :
_access(access) {
_access(access), _leading_membar(NULL) {
GraphKit* kit = access.kit();
DecoratorSet decorators = access.decorators();
@ -139,12 +140,12 @@ public:
// into actual barriers on most machines, but we still need rest of
// compiler to respect ordering.
if (is_release) {
kit->insert_mem_bar(Op_MemBarRelease);
_leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
} else if (is_volatile) {
if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
kit->insert_mem_bar(Op_MemBarVolatile);
_leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
} else {
kit->insert_mem_bar(Op_MemBarRelease);
_leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
}
}
} else if (is_write) {
@ -152,7 +153,7 @@ public:
// floating down past the volatile write. Also prevents commoning
// another volatile read.
if (is_volatile || is_release) {
kit->insert_mem_bar(Op_MemBarRelease);
_leading_membar = kit->insert_mem_bar(Op_MemBarRelease);
}
} else {
// Memory barrier to prevent normal and 'unsafe' accesses from
@ -161,7 +162,7 @@ public:
// so there's no problems making a strong assert about mixing users
// of safe & unsafe memory.
if (is_volatile && support_IRIW_for_not_multiple_copy_atomic_cpu) {
kit->insert_mem_bar(Op_MemBarVolatile);
_leading_membar = kit->insert_mem_bar(Op_MemBarVolatile);
}
}
@ -196,20 +197,30 @@ public:
if (is_atomic) {
if (is_acquire || is_volatile) {
kit->insert_mem_bar(Op_MemBarAcquire);
Node* n = _access.raw_access();
Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
if (_leading_membar != NULL) {
MemBarNode::set_load_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
}
}
} else if (is_write) {
// If not multiple copy atomic, we do the MemBarVolatile before the load.
if (is_volatile && !support_IRIW_for_not_multiple_copy_atomic_cpu) {
kit->insert_mem_bar(Op_MemBarVolatile); // Use fat membar
Node* n = _access.raw_access();
Node* mb = kit->insert_mem_bar(Op_MemBarVolatile, n); // Use fat membar
if (_leading_membar != NULL) {
MemBarNode::set_store_pair(_leading_membar->as_MemBar(), mb->as_MemBar());
}
}
} else {
if (is_volatile || is_acquire) {
kit->insert_mem_bar(Op_MemBarAcquire, _access.raw_access());
Node* n = _access.raw_access();
assert(_leading_membar == NULL || support_IRIW_for_not_multiple_copy_atomic_cpu, "no leading membar expected");
Node* mb = kit->insert_mem_bar(Op_MemBarAcquire, n);
mb->as_MemBar()->set_trailing_load();
}
}
}
};
Node* BarrierSetC2::store_at(C2Access& access, C2AccessValue& val) const {

@ -2767,6 +2767,17 @@ void Compile::final_graph_reshaping_impl( Node *n, Final_Reshape_Counts &frc) {
LoadNode::is_immutable_value(n->in(MemNode::Address))),
"raw memory operations should have control edge");
}
if (n->is_MemBar()) {
MemBarNode* mb = n->as_MemBar();
if (mb->trailing_store() || mb->trailing_load_store()) {
assert(mb->leading_membar()->trailing_membar() == mb, "bad membar pair");
Node* mem = mb->in(MemBarNode::Precedent);
assert((mb->trailing_store() && mem->is_Store() && mem->as_Store()->is_release()) ||
(mb->trailing_load_store() && mem->is_LoadStore()), "missing mem op");
} else if (mb->leading()) {
assert(mb->trailing_membar()->leading_membar() == mb, "bad membar pair");
}
}
#endif
// Count FPU ops and common calls, implements item (3)
switch( nop ) {

@ -2521,45 +2521,63 @@ Node* StoreNode::Identity(PhaseGVN* phase) {
Node* adr = in(MemNode::Address);
Node* val = in(MemNode::ValueIn);
Node* result = this;
// Load then Store? Then the Store is useless
if (val->is_Load() &&
val->in(MemNode::Address)->eqv_uncast(adr) &&
val->in(MemNode::Memory )->eqv_uncast(mem) &&
val->as_Load()->store_Opcode() == Opcode()) {
return mem;
result = mem;
}
// Two stores in a row of the same value?
if (mem->is_Store() &&
if (result == this &&
mem->is_Store() &&
mem->in(MemNode::Address)->eqv_uncast(adr) &&
mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
mem->Opcode() == Opcode()) {
return mem;
result = mem;
}
// Store of zero anywhere into a freshly-allocated object?
// Then the store is useless.
// (It must already have been captured by the InitializeNode.)
if (ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
if (result == this &&
ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
// a newly allocated object is already all-zeroes everywhere
if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
return mem;
result = mem;
}
// the store may also apply to zero-bits in an earlier object
Node* prev_mem = find_previous_store(phase);
// Steps (a), (b): Walk past independent stores to find an exact match.
if (prev_mem != NULL) {
Node* prev_val = can_see_stored_value(prev_mem, phase);
if (prev_val != NULL && phase->eqv(prev_val, val)) {
// prev_val and val might differ by a cast; it would be good
// to keep the more informative of the two.
return mem;
if (result == this) {
// the store may also apply to zero-bits in an earlier object
Node* prev_mem = find_previous_store(phase);
// Steps (a), (b): Walk past independent stores to find an exact match.
if (prev_mem != NULL) {
Node* prev_val = can_see_stored_value(prev_mem, phase);
if (prev_val != NULL && phase->eqv(prev_val, val)) {
// prev_val and val might differ by a cast; it would be good
// to keep the more informative of the two.
result = mem;
}
}
}
}
return this;
if (result != this && phase->is_IterGVN() != NULL) {
MemBarNode* trailing = trailing_membar();
if (trailing != NULL) {
#ifdef ASSERT
const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
#endif
PhaseIterGVN* igvn = phase->is_IterGVN();
trailing->remove(igvn);
}
}
return result;
}
//------------------------------match_edge-------------------------------------
@ -2637,6 +2655,33 @@ bool StoreNode::value_never_loaded( PhaseTransform *phase) const {
return true;
}
MemBarNode* StoreNode::trailing_membar() const {
if (is_release()) {
MemBarNode* trailing_mb = NULL;
for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
Node* u = fast_out(i);
if (u->is_MemBar()) {
if (u->as_MemBar()->trailing_store()) {
assert(u->Opcode() == Op_MemBarVolatile, "");
assert(trailing_mb == NULL, "only one");
trailing_mb = u->as_MemBar();
#ifdef ASSERT
Node* leading = u->as_MemBar()->leading_membar();
assert(leading->Opcode() == Op_MemBarRelease, "incorrect membar");
assert(leading->as_MemBar()->leading_store(), "incorrect membar pair");
assert(leading->as_MemBar()->trailing_membar() == u, "incorrect membar pair");
#endif
} else {
assert(u->as_MemBar()->standalone(), "");
}
}
}
return trailing_mb;
}
return NULL;
}
//=============================================================================
//------------------------------Ideal------------------------------------------
// If the store is from an AND mask that leaves the low bits untouched, then
@ -2749,6 +2794,30 @@ bool LoadStoreNode::result_not_used() const {
return true;
}
MemBarNode* LoadStoreNode::trailing_membar() const {
MemBarNode* trailing = NULL;
for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
Node* u = fast_out(i);
if (u->is_MemBar()) {
if (u->as_MemBar()->trailing_load_store()) {
assert(u->Opcode() == Op_MemBarAcquire, "");
assert(trailing == NULL, "only one");
trailing = u->as_MemBar();
#ifdef ASSERT
Node* leading = trailing->leading_membar();
assert(support_IRIW_for_not_multiple_copy_atomic_cpu || leading->Opcode() == Op_MemBarRelease, "incorrect membar");
assert(leading->as_MemBar()->leading_load_store(), "incorrect membar pair");
assert(leading->as_MemBar()->trailing_membar() == trailing, "incorrect membar pair");
#endif
} else {
assert(u->as_MemBar()->standalone(), "wrong barrier kind");
}
}
}
return trailing;
}
uint LoadStoreNode::size_of() const { return sizeof(*this); }
//=============================================================================
@ -2934,7 +3003,10 @@ Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
//=============================================================================
MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
: MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
_adr_type(C->get_adr_type(alias_idx))
_adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
#ifdef ASSERT
, _pair_idx(0)
#endif
{
init_class_id(Class_MemBar);
Node* top = C->top();
@ -2969,6 +3041,21 @@ MemBarNode* MemBarNode::make(Compile* C, int opcode, int atp, Node* pn) {
}
}
void MemBarNode::remove(PhaseIterGVN *igvn) {
if (outcnt() != 2) {
return;
}
if (trailing_store() || trailing_load_store()) {
MemBarNode* leading = leading_membar();
if (leading != NULL) {
assert(leading->trailing_membar() == this, "inconsistent leading/trailing membars");
leading->remove(igvn);
}
}
igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
}
//------------------------------Ideal------------------------------------------
// Return a node which is more "ideal" than the current node. Strip out
// control copies
@ -3035,8 +3122,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
if (eliminate) {
// Replace MemBar projections by its inputs.
PhaseIterGVN* igvn = phase->is_IterGVN();
igvn->replace_node(proj_out(TypeFunc::Memory), in(TypeFunc::Memory));
igvn->replace_node(proj_out(TypeFunc::Control), in(TypeFunc::Control));
remove(igvn);
// Must return either the original node (now dead) or a new node
// (Do not return a top here, since that would break the uniqueness of top.)
return new ConINode(TypeInt::ZERO);
@ -3065,6 +3151,98 @@ Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
return NULL;
}
void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
trailing->_kind = TrailingStore;
leading->_kind = LeadingStore;
#ifdef ASSERT
trailing->_pair_idx = leading->_idx;
leading->_pair_idx = leading->_idx;
#endif
}
void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
trailing->_kind = TrailingLoadStore;
leading->_kind = LeadingLoadStore;
#ifdef ASSERT
trailing->_pair_idx = leading->_idx;
leading->_pair_idx = leading->_idx;
#endif
}
MemBarNode* MemBarNode::trailing_membar() const {
Node* trailing = (Node*)this;
VectorSet seen(Thread::current()->resource_area());
while (!trailing->is_MemBar() || !trailing->as_MemBar()->trailing()) {
if (seen.test_set(trailing->_idx)) {
// Dying subgraph?
return NULL;
}
for (DUIterator_Fast jmax, j = trailing->fast_outs(jmax); j < jmax; j++) {
Node* next = trailing->fast_out(j);
if (next != trailing && next->is_CFG()) {
trailing = next;
break;
}
}
}
MemBarNode* mb = trailing->as_MemBar();
assert((mb->_kind == TrailingStore && _kind == LeadingStore) ||
(mb->_kind == TrailingLoadStore && _kind == LeadingLoadStore), "bad trailing membar");
assert(mb->_pair_idx == _pair_idx, "bad trailing membar");
return mb;
}
MemBarNode* MemBarNode::leading_membar() const {
VectorSet seen(Thread::current()->resource_area());
Node* leading = in(0);
while (leading != NULL && (!leading->is_MemBar() || !leading->as_MemBar()->leading())) {
if (seen.test_set(leading->_idx)) {
// Dying subgraph?
return NULL;
}
if (leading->is_Region()) {
leading = leading->in(1);
} else {
leading = leading->in(0);
}
}
#ifdef ASSERT
Unique_Node_List wq;
wq.push((Node*)this);
uint found = 0;
for (uint i = 0; i < wq.size(); i++) {
Node* n = wq.at(i);
if (n->is_Region()) {
for (uint j = 1; j < n->req(); j++) {
Node* in = n->in(j);
if (in != NULL && !in->is_top()) {
wq.push(in);
}
}
} else {
if (n->is_MemBar() && n->as_MemBar()->leading()) {
assert(n == leading, "consistency check failed");
found++;
} else {
Node* in = n->in(0);
if (in != NULL && !in->is_top()) {
wq.push(in);
}
}
}
}
assert(found == 1 || (found == 0 && leading == NULL), "consistency check failed");
#endif
if (leading == NULL) {
return NULL;
}
MemBarNode* mb = leading->as_MemBar();
assert((mb->_kind == LeadingStore && _kind == TrailingStore) ||
(mb->_kind == LeadingLoadStore && _kind == TrailingLoadStore), "bad leading membar");
assert(mb->_pair_idx == _pair_idx, "bad leading membar");
return mb;
}
//===========================InitializeNode====================================
// SUMMARY:
// This node acts as a memory barrier on raw memory, after some raw stores.

@ -607,6 +607,8 @@ public:
// have all possible loads of the value stored been optimized away?
bool value_never_loaded(PhaseTransform *phase) const;
MemBarNode* trailing_membar() const;
};
//------------------------------StoreBNode-------------------------------------
@ -816,6 +818,7 @@ public:
virtual const class TypePtr *adr_type() const { return _adr_type; } // returns bottom_type of address
bool result_not_used() const;
MemBarNode* trailing_membar() const;
};
class LoadStoreConditionalNode : public LoadStoreNode {
@ -1142,6 +1145,20 @@ class MemBarNode: public MultiNode {
// Memory type this node is serializing. Usually either rawptr or bottom.
const TypePtr* _adr_type;
// How is this membar related to a nearby memory access?
enum {
Standalone,
TrailingLoad,
TrailingStore,
LeadingStore,
TrailingLoadStore,
LeadingLoadStore
} _kind;
#ifdef ASSERT
uint _pair_idx;
#endif
public:
enum {
Precedent = TypeFunc::Parms // optional edge to force precedence
@ -1159,6 +1176,24 @@ public:
static MemBarNode* make(Compile* C, int opcode,
int alias_idx = Compile::AliasIdxBot,
Node* precedent = NULL);
MemBarNode* trailing_membar() const;
MemBarNode* leading_membar() const;
void set_trailing_load() { _kind = TrailingLoad; }
bool trailing_load() const { return _kind == TrailingLoad; }
bool trailing_store() const { return _kind == TrailingStore; }
bool leading_store() const { return _kind == LeadingStore; }
bool trailing_load_store() const { return _kind == TrailingLoadStore; }
bool leading_load_store() const { return _kind == LeadingLoadStore; }
bool trailing() const { return _kind == TrailingLoad || _kind == TrailingStore || _kind == TrailingLoadStore; }
bool leading() const { return _kind == LeadingStore || _kind == LeadingLoadStore; }
bool standalone() const { return _kind == Standalone; }
static void set_store_pair(MemBarNode* leading, MemBarNode* trailing);
static void set_load_store_pair(MemBarNode* leading, MemBarNode* trailing);
void remove(PhaseIterGVN *igvn);
};
// "Acquire" - no following ref can move before (but earlier refs can