stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8271954: C2: assert(false) failed: Bad graph detected in build_loop_late

Browse Source 
Reviewed-by: roland, kvn
This commit is contained in:
Christian Hagedorn 2021-09-16 11:57:30 +00:00
parent 14dc5178cf
commit c86e24d4be
3 changed files with 442 additions and 14 deletions
src/hotspot/share/opto
loopPredicate.cpploopnode.hpp
test/hotspot/jtreg/compiler/loopopts
TestUnswitchWithSunkNodes.java

111
src/hotspot/share/opto/loopPredicate.cpp

@ -107,8 +107,9 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred,
// Otherwise, the continuation projection is set up to be the false
// projection. This code is also used to clone predicates to cloned loops.
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
int opcode, bool if_cont_is_true_proj) {
Deoptimization::DeoptReason reason, int opcode,
bool if_cont_is_true_proj, Node_List* old_new,
UnswitchingAction unswitching_action) {
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
@ -193,11 +194,40 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
if (use->is_Phi() && use->outcnt() > 0) {
assert(use->in(0) == rgn, "");
_igvn.rehash_node_delayed(use);
use->add_req(use->in(proj_index));
Node* phi_input = use->in(proj_index);
if (unswitching_action == UnswitchingAction::FastLoopCloning
&& !phi_input->is_CFG() && !phi_input->is_Phi() && get_ctrl(phi_input) == uncommon_proj) {
// There are some control dependent nodes on the uncommon projection and we are currently copying predicates
// to the fast loop in loop unswitching (first step, slow loop is processed afterwards). For the fast loop,
// we need to clone all the data nodes in the chain from the phi ('use') up until the node whose control input
// is the uncommon_proj. The slow loop can reuse the old data nodes and thus only needs to update the control
// input to the uncommon_proj (done on the next invocation of this method when 'unswitch_is_slow_loop' is true.
assert(LoopUnswitching, "sanity check");
phi_input = clone_data_nodes_for_fast_loop(phi_input, uncommon_proj, if_uct, old_new);
} else if (unswitching_action == UnswitchingAction::SlowLoopRewiring) {
// Replace phi input for the old predicate path with TOP as the predicate is dying anyways. This avoids the need
// to clone the data nodes again for the slow loop.
assert(LoopUnswitching, "sanity check");
_igvn.replace_input_of(use, proj_index, C->top());
}
use->add_req(phi_input);
has_phi = true;
}
}
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
if (unswitching_action == UnswitchingAction::SlowLoopRewiring) {
// Rewire the control dependent data nodes for the slow loop from the old to the new uncommon projection.
assert(uncommon_proj->outcnt() > 1 && old_new == NULL, "sanity");
for (DUIterator_Fast jmax, j = uncommon_proj->fast_outs(jmax); j < jmax; j++) {
Node* data = uncommon_proj->fast_out(j);
if (!data->is_CFG()) {
_igvn.replace_input_of(data, 0, if_uct);
set_ctrl(data, if_uct);
--j;
--jmax;
}
}
}
if (new_entry == NULL) {
// Attach if_cont to iff
@ -209,9 +239,70 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
return if_cont->as_Proj();
}
// Clone data nodes for the fast loop while creating a new If with create_new_if_for_predicate. Returns the node which is
// used for the uncommon trap phi input.
Node* PhaseIdealLoop::clone_data_nodes_for_fast_loop(Node* phi_input, ProjNode* uncommon_proj, Node* if_uct, Node_List* old_new) {
// Step 1: Clone all nodes on the data chain but do not rewire anything, yet. Keep track of the cloned nodes
// by using the old_new mapping. This mapping is then used in step 2 to rewire the cloned nodes accordingly.
DEBUG_ONLY(uint last_idx = C->unique();)
Unique_Node_List list;
list.push(phi_input);
for (uint j = 0; j < list.size(); j++) {
Node* next = list.at(j);
Node* clone = next->clone();
_igvn.register_new_node_with_optimizer(clone);
old_new->map(next->_idx, clone);
for (uint k = 1; k < next->req(); k++) {
Node* in = next->in(k);
if (!in->is_Phi() && get_ctrl(in) == uncommon_proj) {
list.push(in);
}
}
}
// Step 2: All nodes are cloned. Rewire them by using the old_new mapping.
for (uint j = 0; j < list.size(); j++) {
Node* next = list.at(j);
Node* clone = old_new->at(next->_idx);
assert(clone != NULL && clone->_idx >= last_idx, "must exist and be a proper clone");
if (next->in(0) == uncommon_proj) {
// All data nodes with a control input to the uncommon projection in the chain need to be rewired to the new uncommon
// projection (could not only be the last data node in the chain but also, for example, a DivNode within the chain).
_igvn.replace_input_of(clone, 0, if_uct);
set_ctrl(clone, if_uct);
}
// Rewire the inputs of the cloned nodes to the old nodes to the new clones.
for (uint k = 1; k < next->req(); k++) {
Node* in = next->in(k);
if (!in->is_Phi()) {
assert(!in->is_CFG(), "must be data node");
Node* in_clone = old_new->at(in->_idx);
if (in_clone != NULL) {
assert(in_clone->_idx >= last_idx, "must be a valid clone");
_igvn.replace_input_of(clone, k, in_clone);
set_ctrl(clone, if_uct);
}
}
}
}
Node* clone_phi_input = old_new->at(phi_input->_idx);
assert(clone_phi_input != NULL && clone_phi_input->_idx >= last_idx, "must exist and be a proper clone");
return clone_phi_input;
}
//--------------------------clone_predicate-----------------------
ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason) {
ProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If);
ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, Node_List* old_new) {
UnswitchingAction unswitching_action;
if (predicate_proj->other_if_proj()->outcnt() > 1) {
// There are some data dependencies that need to be taken care of when cloning a predicate.
unswitching_action = old_new == NULL ? UnswitchingAction::SlowLoopRewiring : UnswitchingAction::FastLoopCloning;
} else {
unswitching_action = UnswitchingAction::None;
}
ProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If,
true, old_new, unswitching_action);
IfNode* iff = new_predicate_proj->in(0)->as_If();
Node* ctrl = iff->in(0);
@ -319,7 +410,7 @@ ProjNode* PhaseIdealLoop::clone_skeleton_predicate_for_unswitched_loops(Node* if
//--------------------------clone_loop_predicates-----------------------
// Clone loop predicates to cloned loops when unswitching a loop.
void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred) {
void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred) {
LoopNode* head = loop->_head->as_Loop();
bool clone_limit_check = !head->is_CountedLoop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
@ -343,7 +434,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
}
if (predicate_proj != NULL) { // right pattern that can be used by loop predication
// clone predicate
iffast_pred = clone_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate);
iffast_pred = clone_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate, &old_new);
ifslow_pred = clone_predicate_to_unswitched_loop(predicate_proj, ifslow_pred, Deoptimization::Reason_predicate);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj, iffast_pred, ifslow_pred);
@ -352,7 +443,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
}
if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication
// clone predicate
iffast_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred, Deoptimization::Reason_profile_predicate);
iffast_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred, Deoptimization::Reason_profile_predicate, &old_new);
ifslow_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, ifslow_pred, Deoptimization::Reason_profile_predicate);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate, profile_predicate_proj, iffast_pred, ifslow_pred);
@ -363,7 +454,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
// Clone loop limit check last to insert it before loop.
// Don't clone a limit check which was already finalized
// for this counted loop (only one limit check is needed).
iffast_pred = clone_predicate_to_unswitched_loop(limit_check_proj, iffast_pred, Deoptimization::Reason_loop_limit_check);
iffast_pred = clone_predicate_to_unswitched_loop(limit_check_proj, iffast_pred, Deoptimization::Reason_loop_limit_check, &old_new);
ifslow_pred = clone_predicate_to_unswitched_loop(limit_check_proj, ifslow_pred, Deoptimization::Reason_loop_limit_check);
check_created_predicate_for_unswitching(iffast_pred);
@ -372,7 +463,7 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, co
}
#ifndef PRODUCT
void PhaseIdealLoop::check_created_predicate_for_unswitching(const Node* new_entry) const {
void PhaseIdealLoop::check_created_predicate_for_unswitching(const Node* new_entry) {
assert(new_entry != NULL, "IfTrue or IfFalse after clone predicate");
if (TraceLoopPredicate) {
tty->print("Loop Predicate cloned: ");

20
src/hotspot/share/opto/loopnode.hpp

@ -1275,9 +1275,20 @@ public:
// Return true if exp is a scaled induction var plus (or minus) constant
bool is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth = 0);
// Enum to determine the action to be performed in create_new_if_for_predicate() when processing phis of UCT regions.
enum class UnswitchingAction {
None, // No special action.
FastLoopCloning, // Need to clone nodes for the fast loop.
SlowLoopRewiring // Need to rewire nodes for the slow loop.
};
// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason,
int opcode, bool if_cont_is_true_proj = true);
int opcode, bool if_cont_is_true_proj = true, Node_List* old_new = NULL,
UnswitchingAction unswitching_action = UnswitchingAction::None);
// Clone data nodes for the fast loop while creating a new If with create_new_if_for_predicate.
Node* clone_data_nodes_for_fast_loop(Node* phi_input, ProjNode* uncommon_proj, Node* if_uct, Node_List* old_new);
void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true);
@ -1563,13 +1574,14 @@ private:
}
// Clone loop predicates to slow and fast loop when unswitching a loop
void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred);
ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason);
void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred);
ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason,
Node_List* old_new = NULL);
void clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason,
ProjNode* old_predicate_proj, ProjNode* iffast_pred, ProjNode* ifslow_pred);
ProjNode* clone_skeleton_predicate_for_unswitched_loops(Node* iff, ProjNode* predicate, Node* uncommon_proj, Deoptimization::DeoptReason reason,
ProjNode* output_proj, IdealLoopTree* loop);
void check_created_predicate_for_unswitching(const Node* new_entry) const PRODUCT_RETURN;
static void check_created_predicate_for_unswitching(const Node* new_entry) PRODUCT_RETURN;
bool _created_loop_node;
#ifdef ASSERT

325
test/hotspot/jtreg/compiler/loopopts/TestUnswitchWithSunkNodes.java Normal file

@ -0,0 +1,325 @@
/*
* Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/**
* @test
* @bug 8271954
* @requires vm.compiler2.enabled
* @summary A pinned Cast node on the UCT projection of a predicate is as input for the UCT phi node for the original
* loop, the fast and slow loop resulting in a dominance failure. These data nodes need to be updated while
* unswitching a loop.
* @run main/othervm -Xcomp -XX:-TieredCompilation -XX:CompileCommand=compileonly,compiler.loopopts.TestUnswitchWithSunkNodes::test*
* compiler.loopopts.TestUnswitchWithSunkNodes
*/
package compiler.loopopts;
public class TestUnswitchWithSunkNodes {
static int iFld, iFld2, iFld3 = 1, iFld4 = 1;
static long instanceCount;
static double dFld;
static boolean bFld;
static int iArrFld[] = new int[10];
static {
init(iArrFld);
}
public static void main(String[] strArr) {
// The testcases with Divisions have additional control inputs which need to be taken care of
testNoDiamond();
testNoDiamondDiv();
testWithDiamond();
testWithDiamondDiv1();
testWithDiamondDiv2();
testWithDiamondOneLongOneShortPath();
testWithDiamondComplex();
testWithDiamondComplexDiv();
}
static void testNoDiamond() {
int i, i2 = 10, i21, i22, i24, i26 = 41724, iArr2[] = new int[10];
double d;
float f2;
init(iArr2);
i = 1;
while (++i < 219) {
for (d = 15; 305 > d; ++d) {
if (bFld) {
instanceCount = i2;
i2 = (((i2 + 3) + iFld2) + iFld3) - iFld4;
}
for (f2 = 5; 87 > f2; ++f2) {
i2 = (int) instanceCount;
for (i22 = 1; 2 > i22; i22++) {
if (bFld) {
iArr2[1] += 190L;
}
}
for (i24 = 1; 2 > i24; i24++) {
switch (i) {
case 88:
i26 += (i24);
}
}
}
}
}
}
static void testNoDiamondDiv() {
int i, i2 = 10, i21, i22, i24, i26 = 41724, iArr2[] = new int[10];
double d;
float f2;
init(iArr2);
i = 1;
while (++i < 219) {
for (d = 15; 305 > d; ++d) {
if (bFld) {
instanceCount = i2;
i2 = (((i2 + 3) + iFld2) + iFld3) / iFld4;
}
for (f2 = 5; 87 > f2; ++f2) {
i2 = (int) instanceCount;
for (i22 = 1; 2 > i22; i22++) {
if (bFld) {
iArr2[1] += 190L;
}
}
for (i24 = 1; 2 > i24; i24++) {
switch (i) {
case 88:
i26 += (i24);
}
}
}
}
}
}
static void testWithDiamond() {
int i, i2 = 10, i21, i22, i24, i26 = 41724, iArr2[] = new int[10];
double d;
float f2;
init(iArr2);
i = 1;
while (++i < 219) {
for (d = 15; 305 > d; ++d) {
if (bFld) {
instanceCount = i2;
double d1 = (double) i2;
i2 = (int)((d1 + iFld4) - (d1 + iFld));
}
for (f2 = 5; 87 > f2; ++f2) {
i2 = (int) instanceCount;
for (i22 = 1; 2 > i22; i22++) {
if (bFld) {
iArr2[1] += 190L;
}
}
for (i24 = 1; 2 > i24; i24++) {
switch (i) {
case 88:
i26 += (i24);
}
}
}
}
}
}
static void testWithDiamondDiv1() {
int i, i2 = 10, i21, i22, i24, i26 = 41724, iArr2[] = new int[10];
double d;
float f2;
init(iArr2);
i = 1;
while (++i < 219) {
for (d = 15; 305 > d; ++d) {
if (bFld) {
i2 = (i2 / iFld4) - (i2 / iFld3);
}
for (f2 = 5; 87 > f2; ++f2) {
i2 = (int) instanceCount;
for (i22 = 1; 2 > i22; i22++) {
if (bFld) {
iArr2[1] += 190L;
}
}
for (i24 = 1; 2 > i24; i24++) {
switch (i) {
case 88:
i26 += (i24);
}
}
}
}
}
}
static void testWithDiamondDiv2() {
int i, i2 = 10, i21, i22, i24, i26 = 41724, iArr2[] = new int[10];
double d;
float f2;
init(iArr2);
i = 1;
while (++i < 219) {
for (d = 15; 305 > d; ++d) {
if (bFld) {
instanceCount = i2;
int i3 = (int)d;
i2 = (i3 / iFld4) - (i3 / iFld3);
}
for (f2 = 5; 87 > f2; ++f2) {
i2 = (int) instanceCount;
for (i22 = 1; 2 > i22; i22++) {
if (bFld) {
iArr2[1] += 190L;
}
}
for (i24 = 1; 2 > i24; i24++) {
switch (i) {
case 88:
i26 += (i24);
}
}
}
}
}
}
static void testWithDiamondOneLongOneShortPath() {
int i, i2 = 10, i21, i22, i24, i26 = 41724, iArr2[] = new int[10];
double d;
float f2;
init(iArr2);
i = 1;
while (++i < 219) {
for (d = 15; 305 > d; ++d) {
if (bFld) {
instanceCount = i2;
double d1 = (double)i2;
i2 = (int)((d1 + 3 + iFld2 + iFld3 + iFld4) - (d1 + iFld));
}
for (f2 = 5; 87 > f2; ++f2) {
i2 = (int) instanceCount;
for (i22 = 1; 2 > i22; i22++) {
if (bFld) {
iArr2[1] += 190L;
}
}
for (i24 = 1; 2 > i24; i24++) {
switch (i) {
case 88:
i26 += (i24);
}
}
}
}
}
}
static void testWithDiamondComplex() {
int i, i2 = 10, i21, i22, i24, i26 = 41724, iArr2[] = new int[10];
double d;
float f2;
init(iArr2);
i = 1;
while (++i < 219) {
for (d = 15; 305 > d; ++d) {
if (bFld) {
instanceCount = i2;
i2 += 4;
float l = (float)i2;
i2 = (int)((l + 3) - (l + iFld));
double d1 = (double)i2;
i2 = (int)((d1 + 3) - (d1 + iFld) - (d1 * iFld2));
i2 = (int)((l + 3) - (l + iFld) - (l * iFld2) + (d1 - iFld2) + (d1 / iFld3));
d1 = (double)i2;
i2 = (int)((d1 + 3) - (d1 + iFld) - (d1 * iFld2) + (d1 - iFld2) + (d1 / iFld3));
i2 += dFld;
l = (float)i2;
i2 = (int)((l + 3) - (l + iFld) - (l * iFld2) + (d1 - iFld2) + (d1 / iFld3) - (d1 + iFld) - (d1 * iFld2) + (d1 - iFld2) + (d1 / iFld3));
d1 = (double)i2;
i2 = (int)((d1 + 3) - (d1 + iFld) - (d1 * iFld2) + (d1 - iFld2) + (d1 / iFld3) + (d1 + 3) - (d1 + iFld) - (d1 * iFld2) + (d1 - iFld2) + (d1 / iFld3));
d1 = (double) i2;
i2 = (int) ((d1 + 3 + iFld2 + iFld3 + iFld4) - (d1 + iFld));
}
for (f2 = 5; 87 > f2; ++f2) {
i2 = (int) instanceCount;
for (i22 = 1; 2 > i22; i22++) {
if (bFld) {
iArr2[1] += 190L;
}
}
for (i24 = 1; 2 > i24; i24++) {
switch (i) {
case 88:
i26 += (i24);
}
}
}
}
}
}
static void testWithDiamondComplexDiv() {
int i, i2 = 10, i21, i22, i24, i26 = 41724, iArr2[] = new int[10];
double d;
float f2;
init(iArr2);
i = 1;
while (++i < 219) {
for (d = 15; 305 > d; ++d) {
if (bFld) {
instanceCount = i2;
int i3 = (int)d;
i2 = (i3 / iFld4) - (i3 / iFld3);
double d1 = (double) i2;
i3 = (int)((d1 + iFld4) - (d1 + iFld));
i2 = (i3 / iFld4) - (i3 / iFld3);
}
for (f2 = 5; 87 > f2; ++f2) {
i2 = (int) instanceCount;
for (i22 = 1; 2 > i22; i22++) {
if (bFld) {
iArr2[1] += 190L;
}
}
for (i24 = 1; 2 > i24; i24++) {
switch (i) {
case 88:
i26 += (i24);
}
}
}
}
}
}
public static void init(int[] a) {
for (int j = 0; j < a.length; j++) {
a[j] = j;
}
}
}