stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8251925: C2: RenaissanceStressTest fails with assert(!had_error): bad dominance

Browse Source 
Reviewed-by: kvn, thartmann
This commit is contained in:
Christian Hagedorn 2020-11-23 08:37:09 +00:00
parent 1f32c11d5e
commit e4a32bea9f
2 changed files with 95 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
src/hotspot/share/opto/superword.cpp
View File

@ -69,7 +69,8 @@ SuperWord::SuperWord(PhaseIdealLoop* phase) :
_nlist(arena(), 8, 0, NULL), // scratch list of nodes
_stk(arena(), 8, 0, NULL), // scratch stack of nodes
_lpt(NULL), // loop tree node
_lp(NULL), // LoopNode
_lp(NULL), // CountedLoopNode
_pre_loop_end(NULL), // Pre loop CountedLoopEndNode
_bb(NULL), // basic block
_iv(NULL), // induction var
_race_possible(false), // cases where SDMU is true
@ -155,10 +156,15 @@ void SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) {
if (cl->is_main_loop()) {
// Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
CountedLoopEndNode* pre_end = get_pre_loop_end(cl);
if (pre_end == NULL) return;
Node *pre_opaq1 = pre_end->limit();
if (pre_opaq1->Opcode() != Op_Opaque1) return;
CountedLoopEndNode* pre_end = find_pre_loop_end(cl);
if (pre_end == NULL) {
return;
}
Node* pre_opaq1 = pre_end->limit();
if (pre_opaq1->Opcode() != Op_Opaque1) {
return;
}
set_pre_loop_end(pre_end);
}
init(); // initialize data structures
@ -911,8 +917,7 @@ bool SuperWord::ref_is_alignable(SWPointer& p) {
if (!p.has_iv()) {
return true; // no induction variable
}
CountedLoopEndNode* pre_end = get_pre_loop_end(lp()->as_CountedLoop());
assert(pre_end != NULL, "we must have a correct pre-loop");
CountedLoopEndNode* pre_end = pre_loop_end();
assert(pre_end->stride_is_con(), "pre loop stride is constant");
int preloop_stride = pre_end->stride_con();
@ -3431,21 +3436,19 @@ LoadNode::ControlDependency SuperWord::control_dependency(Node_List* p) {
// to align_to_ref will be a position zero in the vector.
// (iv + k) mod vector_align == 0
void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
CountedLoopNode *main_head = lp()->as_CountedLoop();
assert(main_head->is_main_loop(), "");
CountedLoopEndNode* pre_end = get_pre_loop_end(main_head);
assert(pre_end != NULL, "we must have a correct pre-loop");
Node *pre_opaq1 = pre_end->limit();
assert(lp()->is_main_loop(), "");
CountedLoopEndNode* pre_end = pre_loop_end();
Node* pre_opaq1 = pre_end->limit();
assert(pre_opaq1->Opcode() == Op_Opaque1, "");
Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
Node *lim0 = pre_opaq->in(1);
Opaque1Node* pre_opaq = (Opaque1Node*)pre_opaq1;
Node* lim0 = pre_opaq->in(1);
// Where we put new limit calculations
Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
Node* pre_ctrl = pre_loop_head()->in(LoopNode::EntryControl);
// Ensure the original loop limit is available from the
// pre-loop Opaque1 node.
Node *orig_limit = pre_opaq->original_loop_limit();
Node* orig_limit = pre_opaq->original_loop_limit();
assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, "");
SWPointer align_to_ref_p(align_to_ref, this, NULL, false);
@ -3596,7 +3599,7 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
//----------------------------get_pre_loop_end---------------------------
// Find pre loop end from main loop. Returns null if none.
CountedLoopEndNode* SuperWord::get_pre_loop_end(CountedLoopNode* cl) {
CountedLoopEndNode* SuperWord::find_pre_loop_end(CountedLoopNode* cl) const {
// The loop cannot be optimized if the graph shape at
// the loop entry is inappropriate.
if (!PhaseIdealLoop::is_canonical_loop_entry(cl)) {
@ -3724,7 +3727,7 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal
// Match AddP(base, AddP(ptr, k*iv [+ invariant]), constant)
Node* base = adr->in(AddPNode::Base);
// The base address should be loop invariant
if (!invariant(base)) {
if (is_main_loop_member(base)) {
assert(!valid(), "base address is loop variant");
return;
}
@ -3753,7 +3756,7 @@ SWPointer::SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool anal
break; // stop looking at addp's
}
}
if (!invariant(adr)) {
if (is_main_loop_member(adr)) {
assert(!valid(), "adr is loop variant");
return;
}
@ -3783,12 +3786,23 @@ SWPointer::SWPointer(SWPointer* p) :
#endif
{}
bool SWPointer::is_main_loop_member(Node* n) const {
Node* n_c = phase()->get_ctrl(n);
return lpt()->is_member(phase()->get_loop(n_c));
}
bool SWPointer::invariant(Node* n) {
bool SWPointer::invariant(Node* n) const {
NOT_PRODUCT(Tracer::Depth dd;)
Node *n_c = phase()->get_ctrl(n);
Node* n_c = phase()->get_ctrl(n);
NOT_PRODUCT(_tracer.invariant_1(n, n_c);)
return !lpt()->is_member(phase()->get_loop(n_c));
bool is_not_member = !is_main_loop_member(n);
if (is_not_member && _slp->lp()->is_main_loop()) {
// Check that n_c dominates the pre loop head node. If it does not, then we cannot use n as invariant for the pre loop
// CountedLoopEndNode check because n_c is either part of the pre loop or between the pre and the main loop (illegal
// invariant: Happens, for example, when n_c is a CastII node that prevents data nodes to flow above the main loop).
return phase()->is_dominator(n_c, _slp->pre_loop_head());
}
return is_not_member;
}
//------------------------scaled_iv_plus_offset--------------------
@ -3851,7 +3865,7 @@ bool SWPointer::scaled_iv(Node* n) {
NOT_PRODUCT(_tracer.scaled_iv_3(n, _scale);)
return true;
}
if (_analyze_only && (invariant(n) == false)) {
if (_analyze_only && (is_main_loop_member(n))) {
_nstack->push(n, _stack_idx++);
}
@ -3937,7 +3951,7 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
return false;
}
if (_analyze_only && (invariant(n) == false)) {
if (_analyze_only && is_main_loop_member(n)) {
_nstack->push(n, _stack_idx++);
}
if (opc == Op_AddI) {
@ -3970,20 +3984,27 @@ bool SWPointer::offset_plus_k(Node* n, bool negate) {
return true;
}
}
if (invariant(n)) {
if (!is_main_loop_member(n)) {
// 'n' is loop invariant. Skip range check dependent CastII nodes before checking if 'n' is dominating the pre loop.
if (opc == Op_ConvI2L) {
n = n->in(1);
if (n->Opcode() == Op_CastII &&
n->as_CastII()->has_range_check()) {
// Skip range check dependent CastII nodes
assert(invariant(n), "sanity");
assert(!is_main_loop_member(n), "sanity");
n = n->in(1);
}
// Check if 'n' can really be used as invariant (not in main loop and dominating the pre loop).
if (invariant(n)) {
_negate_invar = negate;
_invar = n;
NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, _negate_invar, _offset);)
return true;
}
}
_negate_invar = negate;
_invar = n;
NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, _negate_invar, _offset);)
return true;
return false;
}
NOT_PRODUCT(_tracer.offset_plus_k_11(n);)
@ -4004,8 +4025,10 @@ void SWPointer::print() {
//----------------------------tracing------------------------
#ifndef PRODUCT
void SWPointer::Tracer::print_depth() {
for (int ii = 0; ii<_depth; ++ii) tty->print(" ");
void SWPointer::Tracer::print_depth() const {
for (int ii = 0; ii < _depth; ++ii) {
tty->print(" ");
}
}
void SWPointer::Tracer::ctor_1 (Node* mem) {
@ -4057,7 +4080,7 @@ void SWPointer::Tracer::ctor_6(Node* mem) {
}
}
void SWPointer::Tracer::invariant_1(Node *n, Node *n_c) {
void SWPointer::Tracer::invariant_1(Node *n, Node *n_c) const {
if (_slp->do_vector_loop() && _slp->is_debug() && _slp->_lpt->is_member(_slp->_phase->get_loop(n_c)) != (int)_slp->in_bb(n)) {
int is_member = _slp->_lpt->is_member(_slp->_phase->get_loop(n_c));
int in_bb = _slp->in_bb(n);

57
src/hotspot/share/opto/superword.hpp
View File

@ -291,11 +291,11 @@ class SuperWord : public ResourceObj {
void unrolling_analysis(int &local_loop_unroll_factor);
// Accessors for SWPointer
PhaseIdealLoop* phase() { return _phase; }
IdealLoopTree* lpt() { return _lpt; }
PhiNode* iv() { return _iv; }
PhaseIdealLoop* phase() const { return _phase; }
IdealLoopTree* lpt() const { return _lpt; }
PhiNode* iv() const { return _iv; }
bool early_return() { return _early_return; }
bool early_return() const { return _early_return; }
#ifndef PRODUCT
bool is_debug() { return _vector_loop_debug > 0; }
@ -310,7 +310,8 @@ class SuperWord : public ResourceObj {
bool do_reserve_copy() { return _do_reserve_copy; }
private:
IdealLoopTree* _lpt; // Current loop tree node
LoopNode* _lp; // Current LoopNode
CountedLoopNode* _lp; // Current CountedLoopNode
CountedLoopEndNode* _pre_loop_end; // Current CountedLoopEndNode of pre loop
Node* _bb; // Current basic block
PhiNode* _iv; // Induction var
bool _race_possible; // In cases where SDMU is true
@ -330,14 +331,33 @@ class SuperWord : public ResourceObj {
Arena* arena() { return _arena; }
Node* bb() { return _bb; }
void set_bb(Node* bb) { _bb = bb; }
void set_bb(Node* bb) { _bb = bb; }
void set_lpt(IdealLoopTree* lpt) { _lpt = lpt; }
CountedLoopNode* lp() const { return _lp; }
void set_lp(CountedLoopNode* lp) {
_lp = lp;
_iv = lp->as_CountedLoop()->phi()->as_Phi();
}
int iv_stride() const { return lp()->stride_con(); }
LoopNode* lp() { return _lp; }
void set_lp(LoopNode* lp) { _lp = lp;
_iv = lp->as_CountedLoop()->phi()->as_Phi(); }
int iv_stride() { return lp()->as_CountedLoop()->stride_con(); }
CountedLoopNode* pre_loop_head() const {
assert(_pre_loop_end != NULL && _pre_loop_end->loopnode() != NULL, "should find head from pre loop end");
return _pre_loop_end->loopnode();
}
void set_pre_loop_end(CountedLoopEndNode* pre_loop_end) {
assert(pre_loop_end, "must be valid");
_pre_loop_end = pre_loop_end;
}
CountedLoopEndNode* pre_loop_end() const {
#ifdef ASSERT
assert(_lp != NULL, "sanity");
assert(_pre_loop_end != NULL, "should be set when fetched");
Node* found_pre_end = find_pre_loop_end(_lp);
assert(_pre_loop_end == found_pre_end && _pre_loop_end == pre_loop_head()->loopexit(),
"should find the pre loop end and must be the same result");
#endif
return _pre_loop_end;
}
int vector_width(Node* n) {
BasicType bt = velt_basic_type(n);
@ -531,7 +551,7 @@ class SuperWord : public ResourceObj {
// to align_to_ref will be a position zero in the vector.
void align_initial_loop_index(MemNode* align_to_ref);
// Find pre loop end from main loop. Returns null if none.
CountedLoopEndNode* get_pre_loop_end(CountedLoopNode *cl);
CountedLoopEndNode* find_pre_loop_end(CountedLoopNode *cl) const;
// Is the use of d1 in u1 at the same operand position as d2 in u2?
bool opnd_positions_match(Node* d1, Node* u1, Node* d2, Node* u2);
void init();
@ -567,11 +587,12 @@ class SWPointer {
bool _analyze_only; // Used in loop unrolling only for swpointer trace
uint _stack_idx; // Used in loop unrolling only for swpointer trace
PhaseIdealLoop* phase() { return _slp->phase(); }
IdealLoopTree* lpt() { return _slp->lpt(); }
PhiNode* iv() { return _slp->iv(); } // Induction var
PhaseIdealLoop* phase() const { return _slp->phase(); }
IdealLoopTree* lpt() const { return _slp->lpt(); }
PhiNode* iv() const { return _slp->iv(); } // Induction var
bool invariant(Node* n);
bool is_main_loop_member(Node* n) const;
bool invariant(Node* n) const;
// Match: k*iv + offset
bool scaled_iv_plus_offset(Node* n);
@ -638,7 +659,7 @@ class SWPointer {
SuperWord* _slp;
static int _depth;
int _depth_save;
void print_depth();
void print_depth() const;
int depth() const { return _depth; }
void set_depth(int d) { _depth = d; }
void inc_depth() { _depth++;}
@ -664,7 +685,7 @@ class SWPointer {
void ctor_5(Node* adr, Node* base, int i);
void ctor_6(Node* mem);
void invariant_1(Node *n, Node *n_c);
void invariant_1(Node *n, Node *n_c) const;
void scaled_iv_plus_offset_1(Node* n);
void scaled_iv_plus_offset_2(Node* n);