stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8305634: Renaming predicates, simple cleanups, and adding summary about current predicates

Browse Source 
Reviewed-by: epeter, thartmann, roland
This commit is contained in:
Christian Hagedorn 2023-05-16 13:30:07 +00:00
parent 72294c5402
commit 19c8c30d1c
19 changed files with 685 additions and 505 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/hotspot/share/opto/cfgnode.cpp
View File

@ -1796,7 +1796,7 @@ static Node* split_flow_path(PhaseGVN *phase, PhiNode *phi) {
if( phase->type(n) == Type::TOP ) return nullptr;
if( phi->in(i) == val ) {
hit++;
if (PhaseIdealLoop::find_predicate(r->in(i)) != nullptr) {
if (PhaseIdealLoop::find_parse_predicate(r->in(i)) != nullptr) {
return nullptr; // don't split loop entry path
}
}

4
src/hotspot/share/opto/cfgnode.hpp
View File

@ -58,6 +58,10 @@ class JumpProjNode;
class SCMemProjNode;
class PhaseIdealLoop;
// The success projection of a Parse Predicate is always an IfTrueNode and the uncommon projection an IfFalseNode
typedef IfTrueNode ParsePredicateSuccessProj;
typedef IfFalseNode ParsePredicateUncommonProj;
//------------------------------RegionNode-------------------------------------
// The class of RegionNodes, which can be mapped to basic blocks in the
// program. Their inputs point to Control sources. PhiNodes (described

27
src/hotspot/share/opto/compile.cpp
View File

@ -387,7 +387,7 @@ void Compile::remove_useless_node(Node* dead) {
remove_expensive_node(dead);
}
if (dead->Opcode() == Op_Opaque4) {
remove_skeleton_predicate_opaq(dead);
remove_template_assertion_predicate_opaq(dead);
}
if (dead->for_post_loop_opts_igvn()) {
remove_from_post_loop_opts_igvn(dead);
@ -435,8 +435,8 @@ void Compile::disconnect_useless_nodes(Unique_Node_List &useful, Unique_Node_Lis
}
remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
remove_useless_nodes(_predicate_opaqs, useful); // remove useless predicate opaque nodes
remove_useless_nodes(_skeleton_predicate_opaqs, useful);
remove_useless_nodes(_parse_predicate_opaqs, useful); // remove useless Parse Predicate opaque nodes
remove_useless_nodes(_template_assertion_predicate_opaqs, useful); // remove useless Assertion Predicate opaque nodes
remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
remove_useless_unstable_if_traps(useful); // remove useless unstable_if traps
@ -615,8 +615,8 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci,
_failure_reason(nullptr),
_intrinsics (comp_arena(), 0, 0, nullptr),
_macro_nodes (comp_arena(), 8, 0, nullptr),
_predicate_opaqs (comp_arena(), 8, 0, nullptr),
_skeleton_predicate_opaqs (comp_arena(), 8, 0, nullptr),
_parse_predicate_opaqs (comp_arena(), 8, 0, nullptr),
_template_assertion_predicate_opaqs (comp_arena(), 8, 0, nullptr),
_expensive_nodes (comp_arena(), 8, 0, nullptr),
_for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
_unstable_if_traps (comp_arena(), 8, 0, nullptr),
@ -1792,17 +1792,18 @@ bool Compile::can_alias(const TypePtr* adr_type, int alias_idx) {
return adr_idx == alias_idx;
}
//---------------------cleanup_loop_predicates-----------------------
// Remove the opaque nodes that protect the predicates so that all unused
// checks and uncommon_traps will be eliminated from the ideal graph
void Compile::cleanup_loop_predicates(PhaseIterGVN &igvn) {
if (predicate_count()==0) return;
for (int i = predicate_count(); i > 0; i--) {
Node * n = predicate_opaque1_node(i-1);
// Remove the opaque nodes that protect the Parse Predicates so that all unused
// checks and uncommon_traps will be eliminated from the ideal graph.
void Compile::cleanup_parse_predicates(PhaseIterGVN& igvn) const {
if (parse_predicate_count() == 0) {
return;
}
for (int i = parse_predicate_count(); i > 0; i--) {
Node* n = parse_predicate_opaque1_node(i - 1);
assert(n->Opcode() == Op_Opaque1, "must be");
igvn.replace_node(n, n->in(1));
}
assert(predicate_count()==0, "should be clean!");
assert(parse_predicate_count() == 0, "should be clean!");
}
void Compile::record_for_post_loop_opts_igvn(Node* n) {

49
src/hotspot/share/opto/compile.hpp
View File

@ -351,8 +351,8 @@ class Compile : public Phase {
const char* _failure_reason; // for record_failure/failing pattern
GrowableArray<CallGenerator*> _intrinsics; // List of intrinsics.
GrowableArray<Node*> _macro_nodes; // List of nodes which need to be expanded before matching.
GrowableArray<Node*> _predicate_opaqs; // List of Opaque1 nodes for the loop predicates.
GrowableArray<Node*> _skeleton_predicate_opaqs; // List of Opaque4 nodes for the loop skeleton predicates.
GrowableArray<Node*> _parse_predicate_opaqs; // List of Opaque1 nodes for the Parse Predicates.
GrowableArray<Node*> _template_assertion_predicate_opaqs; // List of Opaque4 nodes for Template Assertion Predicates.
GrowableArray<Node*> _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common
GrowableArray<Node*> _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over
GrowableArray<UnstableIfTrap*> _unstable_if_traps; // List of ifnodes after IGVN
@ -668,14 +668,18 @@ class Compile : public Phase {
#endif
int macro_count() const { return _macro_nodes.length(); }
int predicate_count() const { return _predicate_opaqs.length(); }
int skeleton_predicate_count() const { return _skeleton_predicate_opaqs.length(); }
int parse_predicate_count() const { return _parse_predicate_opaqs.length(); }
int template_assertion_predicate_count() const { return _template_assertion_predicate_opaqs.length(); }
int expensive_count() const { return _expensive_nodes.length(); }
int coarsened_count() const { return _coarsened_locks.length(); }
Node* macro_node(int idx) const { return _macro_nodes.at(idx); }
Node* predicate_opaque1_node(int idx) const { return _predicate_opaqs.at(idx); }
Node* skeleton_predicate_opaque4_node(int idx) const { return _skeleton_predicate_opaqs.at(idx); }
Node* parse_predicate_opaque1_node(int idx) const { return _parse_predicate_opaqs.at(idx); }
Node* template_assertion_predicate_opaq_node(int idx) const {
return _template_assertion_predicate_opaqs.at(idx);
}
Node* expensive_node(int idx) const { return _expensive_nodes.at(idx); }
ConnectionGraph* congraph() { return _congraph;}
@ -689,9 +693,9 @@ class Compile : public Phase {
// this function may be called twice for a node so we can only remove it
// if it's still existing.
_macro_nodes.remove_if_existing(n);
// remove from _predicate_opaqs list also if it is there
if (predicate_count() > 0) {
_predicate_opaqs.remove_if_existing(n);
// remove from _parse_predicate_opaqs list also if it is there
if (parse_predicate_count() > 0) {
_parse_predicate_opaqs.remove_if_existing(n);
}
// Remove from coarsened locks list if present
if (coarsened_count() > 0) {
@ -702,18 +706,19 @@ class Compile : public Phase {
void remove_expensive_node(Node* n) {
_expensive_nodes.remove_if_existing(n);
}
void add_predicate_opaq(Node* n) {
assert(!_predicate_opaqs.contains(n), "duplicate entry in predicate opaque1");
void add_parse_predicate_opaq(Node* n) {
assert(!_parse_predicate_opaqs.contains(n), "duplicate entry in Parse Predicate opaque1 list");
assert(_macro_nodes.contains(n), "should have already been in macro list");
_predicate_opaqs.append(n);
_parse_predicate_opaqs.append(n);
}
void add_skeleton_predicate_opaq(Node* n) {
assert(!_skeleton_predicate_opaqs.contains(n), "duplicate entry in skeleton predicate opaque4 list");
_skeleton_predicate_opaqs.append(n);
void add_template_assertion_predicate_opaq(Node* n) {
assert(!_template_assertion_predicate_opaqs.contains(n),
"duplicate entry in template assertion predicate opaque4 list");
_template_assertion_predicate_opaqs.append(n);
}
void remove_skeleton_predicate_opaq(Node* n) {
if (skeleton_predicate_count() > 0) {
_skeleton_predicate_opaqs.remove_if_existing(n);
void remove_template_assertion_predicate_opaq(Node* n) {
if (template_assertion_predicate_count() > 0) {
_template_assertion_predicate_opaqs.remove_if_existing(n);
}
}
void add_coarsened_locks(GrowableArray<AbstractLockNode*>& locks);
@ -735,11 +740,11 @@ class Compile : public Phase {
void sort_macro_nodes();
// remove the opaque nodes that protect the predicates so that the unused checks and
// Remove the opaque nodes that protect the Parse Predicates so that the unused checks and
// uncommon traps will be eliminated from the graph.
void cleanup_loop_predicates(PhaseIterGVN &igvn);
bool is_predicate_opaq(Node* n) {
return _predicate_opaqs.contains(n);
void cleanup_parse_predicates(PhaseIterGVN &igvn) const;
bool is_predicate_opaq(Node* n) const {
return _parse_predicate_opaqs.contains(n);
}
// Are there candidate expensive nodes for optimization?

32
src/hotspot/share/opto/graphKit.cpp
View File

@ -3977,10 +3977,8 @@ InitializeNode* AllocateNode::initialization() {
return nullptr;
}
//----------------------------- loop predicates ---------------------------
//------------------------------add_predicate_impl----------------------------
void GraphKit::add_empty_predicate_impl(Deoptimization::DeoptReason reason, int nargs) {
// Add a Parse Predicate with an uncommon trap on the failing/false path. Normal control will continue on the true path.
void GraphKit::add_parse_predicate(Deoptimization::DeoptReason reason, const int nargs) {
// Too many traps seen?
if (too_many_traps(reason)) {
#ifdef ASSERT
@ -3993,16 +3991,16 @@ void GraphKit::add_empty_predicate_impl(Deoptimization::DeoptReason reason, int
}
#endif
// We cannot afford to take more traps here,
// do not generate predicate.
// do not generate Parse Predicate.
return;
}
Node *cont = _gvn.intcon(1);
Node* cont = _gvn.intcon(1);
Node* opq = _gvn.transform(new Opaque1Node(C, cont));
Node *bol = _gvn.transform(new Conv2BNode(opq));
Node* bol = _gvn.transform(new Conv2BNode(opq));
IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
Node* iffalse = _gvn.transform(new IfFalseNode(iff));
C->add_predicate_opaq(opq);
C->add_parse_predicate_opaq(opq);
{
PreserveJVMState pjvms(this);
set_control(iffalse);
@ -4013,19 +4011,17 @@ void GraphKit::add_empty_predicate_impl(Deoptimization::DeoptReason reason, int
set_control(iftrue);
}
//------------------------------add_predicate---------------------------------
void GraphKit::add_empty_predicates(int nargs) {
// These loop predicates remain empty. All concrete loop predicates are inserted above the corresponding
// empty loop predicate later by 'PhaseIdealLoop::create_new_if_for_predicate'. All concrete loop predicates of
// a specific kind (normal, profile or limit check) share the same uncommon trap as the empty loop predicate.
// Add Parse Predicates which serve as placeholders to create new Runtime Predicates above them. All
// Runtime Predicates inside a Runtime Predicate block share the same uncommon trap as the Parse Predicate.
void GraphKit::add_parse_predicates(int nargs) {
if (UseLoopPredicate) {
add_empty_predicate_impl(Deoptimization::Reason_predicate, nargs);
add_parse_predicate(Deoptimization::Reason_predicate, nargs);
}
if (UseProfiledLoopPredicate) {
add_empty_predicate_impl(Deoptimization::Reason_profile_predicate, nargs);
add_parse_predicate(Deoptimization::Reason_profile_predicate, nargs);
}
// loop's limit check predicate should be near the loop.
add_empty_predicate_impl(Deoptimization::Reason_loop_limit_check, nargs);
// Loop Limit Check Predicate should be near the loop.
add_parse_predicate(Deoptimization::Reason_loop_limit_check, nargs);
}
void GraphKit::sync_kit(IdealKit& ideal) {
@ -4149,7 +4145,7 @@ void GraphKit::inflate_string_slow(Node* src, Node* dst, Node* start, Node* coun
* dst[i_char++] = (char)(src[i_byte] & 0xff);
* }
*/
add_empty_predicates();
add_parse_predicates();
C->set_has_loops(true);
RegionNode* head = new RegionNode(3);

4
src/hotspot/share/opto/graphKit.hpp
View File

@ -897,8 +897,8 @@ class GraphKit : public Phase {
return iff;
}
void add_empty_predicates(int nargs = 0);
void add_empty_predicate_impl(Deoptimization::DeoptReason reason, int nargs);
void add_parse_predicates(int nargs = 0);
void add_parse_predicate(Deoptimization::DeoptReason reason, int nargs);
Node* make_constant_from_field(ciField* field, Node* obj);

4
src/hotspot/share/opto/idealKit.cpp
View File

@ -167,8 +167,8 @@ void IdealKit::loop(GraphKit* gkit, int nargs, IdealVariable& iv, Node* init, Bo
if (UseLoopPredicate) {
// Sync IdealKit and graphKit.
gkit->sync_kit(*this);
// Add loop predicate.
gkit->add_empty_predicates(nargs);
// Add Parse Predicates.
gkit->add_parse_predicates(nargs);
// Update IdealKit memory.
sync_kit(gkit);
}

2
src/hotspot/share/opto/ifnode.cpp
View File

@ -242,7 +242,7 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) {
if (phi->in(ii) == con1) {
req_c++;
}
Node* proj = PhaseIdealLoop::find_predicate(r->in(ii));
Node* proj = PhaseIdealLoop::find_parse_predicate(r->in(ii));
if (proj != nullptr) {
// Bail out if splitting through a region with a predicate input (could
// also be a loop header before loop opts creates a LoopNode for it).

476
src/hotspot/share/opto/loopPredicate.cpp
View File

@ -40,22 +40,178 @@
#include <math.h>
/*
* The general idea of Loop Predication is to insert a predicate on the entry
* path to a loop, and raise a uncommon trap if the check of the condition fails.
* The condition checks are promoted from inside the loop body, and thus
* the checks inside the loop could be eliminated. Currently, loop predication
* optimization has been applied to remove array range check and loop invariant
* checks (such as null checks).
* The general idea of Loop Predication is to hoist a check inside a loop body by inserting a Hoisted Predicate with an
* uncommon trap on the entry path to the loop. The old check inside the loop can be eliminated. If the condition of the
* Hoisted Predicate fails at runtime, we'll execute the uncommon trap to avoid entering the loop which misses the check.
* Loop Predication can currently remove array range checks and loop invariant checks (such as null checks).
*
* There are at least 3 kinds of predicates: a place holder inserted
* at parse time, the tests added by predication above the place
* holder (referred to as concrete predicates), skeleton predicates
* that are added between main loop and pre loop to protect C2 from
* inconsistencies in some rare cases of over unrolling. Skeleton
* predicates themselves are expanded and updated as unrolling
* proceeds. They don't compile to any code.
* On top of these predicates added by Loop Predication, there are other kinds of predicates. The following list provides
* a complete description of all predicates used in the C2 compiler:
*
*/
*
* There are different kinds of predicates throughout the code. We differentiate between the following predicates:
*
* - Regular Predicate: This term is used to refer to a Parse Predicate or a Runtime Predicate and can be used to
* distinguish from any Assertion Predicate.
* - Parse Predicate: Added during parsing to capture the current JVM state. This predicate represents a "placeholder"
* above which more Runtime Predicates can be created later after parsing.
*
* There are initially three Parse Predicates for each loop:
* - Loop Parse Predicate: The Parse Predicate added for Loop Predicates.
* - Profiled Loop Parse Predicate: The Parse Predicate added for Profiled Loop Predicates.
* - Loop Limit Check Parse Predicate: The Parse Predicate added for a Loop Limit Check Predicate.
* - Runtime Predicate: This term is used to refer to a Hoisted Predicate (either a Loop Predicate or a Profiled Loop
* Predicate) or a Loop Limit Check Predicate. These predicates will be checked at runtime while the
* Parse and Assertion Predicates are always removed before code generation (except for Initialized
* Assertion Predicates which are kept in debug builds while being removed in product builds).
* - Hoisted Predicate: Either a Loop Predicate or a Profiled Loop Predicate that was created during Loop Predication
* to hoist a check out of a loop. Each Hoisted Predicate is accompanied by additional
* Assertion Predicates.
* - Loop Predicate: A predicate that can either hoist a loop-invariant check out of a loop or a range check
* of the form "a[i*scale + offset]", where scale and offset are loop-invariant, out of a
* counted loop. A check must be executed in each loop iteration to hoist it. Otherwise, no
* Loop Predicate can be created. This predicate is created during Loop Predication and is
* inserted above the Loop Parse Predicate.
* - Profiled Loop: This predicate is very similar to a Loop Predicate but the hoisted check does not need
* Predicate to be executed in each loop iteration. By using profiling information, only checks with
* a high execution frequency are chosen to be replaced by a Profiled Loop Predicate. This
* predicate is created during Loop Predication and is inserted above the Profiled Loop
* Parse Predicate.
* - Loop Limit Check: This predicate is created when transforming a loop to a counted loop to protect against
* Predicate the case when adding the stride to the induction variable would cause an overflow which
* will not satisfy the loop limit exit condition. This overflow is unexpected for further
* counted loop optimizations and could lead to wrong results. Therefore, when this predicate
* fails at runtime, we must trap and recompile the method without turning the loop into a
* a counted loop to avoid these overflow problems.
* The predicate does not replace an actual check inside the loop. This predicate can only
* be added once above the Loop Limit Check Parse Predicate for a loop.
* - Assertion Predicate: An always true predicate which will never fail (its range is already covered by an earlier
* Hoisted Predicate or the main-loop entry guard) but is required in order to fold away a dead
* sub loop inside which some data could be proven to be dead (by the type system) and replaced
* by top. Without such Assertion Predicates, we could find that type ranges in Cast and ConvX2Y
* data nodes become impossible and are replaced by top. This is an indicator that the sub loop
* is never executed and must be dead. But there is no way for C2 to prove that the sub loop is
* actually dead. Assertion Predicates come to the rescue to fold such seemingly dead sub loops
* away to avoid a broken graph. Assertion Predicates are left in the graph as a sanity checks in
* debug builds (they must never fail at runtime) while they are being removed in product builds.
* We use special Opaque4 nodes to block some optimizations and replace the Assertion Predicates
* later in product builds.
*
* There are two kinds of Assertion Predicates:
* - Template Assertion Predicate: A template for an Assertion Predicate that uses OpaqueLoop*
* nodes as placeholders for the init and stride value of a loop.
* This predicate does not represent an actual check, yet, and
* just serves as a template to create an Initialized Assertion
* Predicate for a (sub) loop.
* - Initialized Assertion Predicate: An Assertion Predicate that represents an actual check for a
* (sub) loop that was initialized by cloning a Template
* Assertion Predicate. The check is always true and is covered
* by an earlier check (a Hoisted Predicate or the main-loop
* entry guard).
*
* Assertion Predicates are required when removing a range check from a loop. These are inserted
* either at Loop Predication or at Range Check Elimination:
* - Loop Predication: A range check inside a loop is replaced by a Hoisted Predicate before
* the loop. We add two additional Template Assertion Predicates from
* which we can later create Initialized Assertion Predicates. One
* would have been enough if the number of array accesses inside a sub
* loop does not change. But when unrolling the sub loop, we are
* doubling the number of array accesses - we need to cover them all.
* To do that, we only need to create an Initialized Assertion Predicate
* for the first, initial value and for the last value:
* Let a[i] be an array access in the original, not-yet unrolled loop
* with stride 1. When unrolling this loop, we double the stride
* (i.e. stride 2) and have now two accesses a[i] and a[i+1]. We need
* checks for both. When further unrolling this loop, we only need to
* keep the checks on the first and last access (e.g. a[i] and a[i+3]
* on the next unrolling step as they cover the checks in the middle
* for a[i+1] and a[i+2]).
* Therefore, we just need to cover:
* - Initial value: a[init]
* - Last value: a[init + new stride - original stride]
* (We could still only use one Template Assertion Predicate to create
* both Initialized Assertion Predicates from - might be worth doing
* at some point).
* When later splitting a loop (pre/main/post, peeling, unrolling),
* we create two Initialized Assertion Predicates from the Template
* Assertion Predicates by replacing the OpaqueLoop* nodes by actual
* values. Initially (before unrolling), both Assertion Predicates are
* equal. The Initialized Assertion Predicates are always true because
* their range is covered by a corresponding Hoisted Predicate.
* - Range Check Elimination: A range check is removed from the main-loop by changing the pre
* and main-loop iterations. We add two additional Template Assertion
* Predicates (see explanation in section above) and one Initialized
* Assertion Predicate for the just removed range check. When later
* unrolling the main-loop, we create two Initialized Assertion
* Predicates from the Template Assertion Predicates by replacing the
* OpaqueLoop* nodes by actual values for the unrolled loop.
* The Initialized Assertion Predicates are always true: They are true
* when entering the main-loop (because we adjusted the pre-loop exit
* condition), when executing the last iteration of the main-loop
* (because we adjusted the main-loop exit condition), and during all
* other iterations of the main-loop in-between by implication.
* Note that Range Check Elimination could remove additional range
* checks which we were not possible to remove with Loop Predication
* before (for example, because no Parse Predicates were available
* before the loop to create Hoisted Predicates with).
*
*
* In order to group predicates and refer to them throughout the code, we introduce the following additional terms:
* - Regular Predicate Block: A Regular Predicate Block groups all Runtime Predicates in a Runtime Predicate Block
* together with their dedicated Parse Predicate from which they were created (all predicates
* share the same uncommon trap). The Runtime Predicate Block could be empty (i.e. no
* Runtime Predicates created) and the Parse Predicate could be missing (after removing Parse
* Predicates). There are three such Regular Predicate Blocks:
* - Loop Predicate Block
* - Profiled Loop Predicate Block
* - Loop Limit Check Predicate Block
* - Runtime Predicate Block: A block containing all Runtime Predicates that share the same uncommon trap (i.e. belonging
* to a single Parse Predicate which is not included in this block). This block could be empty
* if there were no Runtime Predicates created with the Parse Predicate below this block.
* For the time being: We also count Assertion Predicates to this block but that will be
* changed with the redesign of Assertion Predicates where we remove them from this block
* (JDK-8288981).
*
* Initially, before applying any loop-splitting optimizations, we find the following structure after Loop Predication
* (predicates inside square brackets [] do not need to exist if there are no checks to hoist):
*
* [Loop Hoisted Predicate 1 + two Template Assertion Predicates] \ Runtime \
* [Loop Hoisted Predicate 2 + two Template Assertion Predicates] | Predicate |
* ... | Block | Loop Predicate Block
* [Loop Hoisted Predicate n + two Template Assertion Predicates] / |
* Loop Parse Predicate /
*
* [Profiled Loop Hoisted Predicate 1 + two Template Assertion Predicates] \ Runtime \
* [Profiled Loop Hoisted Predicate 2 + two Template Assertion Predicates] | Predicate | Profiled Loop
* ... | Block | Predicate Block
* [Profiled Loop Hoisted Predicate m + two Template Assertion Predicates] / |
* Profiled Loop Parse Predicate /
* \ Runtime
* [Loop Limit Check Predicate] (at most one) / Predicate \ Loop Limit Check
* Loop Limit Check Parse Predicate Block / Predicate Block
* Loop Head
*
* As an example, let's look at how the predicate structure looks for the main-loop after creating pre/main/post loops
* and applying Range Check Elimination (the order is insignificant):
*
* Main Loop entry (zero-trip) guard
* [For Loop Predicate 1: Two Template + two Initialized Assertion Predicates]
* [For Loop Predicate 2: Two Template + two Initialized Assertion Predicates]
* ...
* [For Loop Predicate n: Two Template + two Initialized Assertion Predicates]
*
* [For Profiled Loop Predicate 1: Two Template + two Initialized Assertion Predicates]
* [For Profiled Loop Predicate 2: Two Template + two Initialized Assertion Predicates]
* ...
* [For Profiled Loop Predicate m: Two Template + two Initialized Assertion Predicates]
*
* (after unrolling, we have two Initialized Assertion Predicates for the Assertion Predicates of Range Check Elimination)
* [For Range Check Elimination Check 1: Two Templates + one Initialized Assertion Predicate]
* [For Range Check Elimination Check 2: Two Templates + one Initialized Assertion Predicate]
* ...
* [For Range Check Elimination Check k: Two Templates + one Initialized Assertion Predicate]
* Main Loop Head
*/
//-------------------------------register_control-------------------------
void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body) {
@ -105,13 +261,11 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred,
//
// We will create a region to guard the uct call if there is no one there.
// The continuation projection (if_cont) of the new_iff is returned which
// is by default a true projection if 'if_cont_is_true_proj' is true.
// Otherwise, the continuation projection is set up to be the false
// projection. This code is also used to clone predicates to cloned loops.
ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
const int opcode, const bool rewire_uncommon_proj_phi_inputs,
const bool if_cont_is_true_proj) {
// is an IfTrue projection. This code is also used to clone predicates to cloned loops.
IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
const int opcode, const bool rewire_uncommon_proj_phi_inputs,
const bool if_cont_is_true_proj) {
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
@ -160,8 +314,8 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
}
register_control(new_iff, lp, entry);
Node* if_cont;
Node* if_uct;
IfProjNode* if_cont;
IfProjNode* if_uct;
if (if_cont_is_true_proj) {
if_cont = new IfTrueNode(new_iff);
if_uct = new IfFalseNode(new_iff);
@ -172,7 +326,7 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
if (cont_proj->is_IfFalse()) {
// Swap
Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
IfProjNode* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
}
register_control(if_cont, lp, new_iff);
register_control(if_uct, get_loop(rgn), new_iff);
@ -223,7 +377,7 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
set_idom(iff, if_cont, dom_depth(iff));
}
}
return if_cont->as_Proj();
return if_cont->as_IfProj();
}
// Update ctrl and control inputs of all data nodes starting from 'node' to 'new_ctrl' which have 'old_ctrl' as
@ -312,19 +466,19 @@ void PhaseIdealLoop::rewire_inputs_of_clones_to_clones(Node* new_ctrl, Node* clo
}
}
//--------------------------clone_predicate-----------------------
ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, const bool slow_loop) {
IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* predicate_proj,
Node* new_entry, Deoptimization::DeoptReason reason,
const bool slow_loop) {
ProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If,
slow_loop);
IfProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If,
slow_loop);
IfNode* iff = new_predicate_proj->in(0)->as_If();
Node* ctrl = iff->in(0);
// Match original condition since predicate's projections could be swapped.
assert(predicate_proj->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
Node* opq = new Opaque1Node(C, predicate_proj->in(0)->in(1)->in(1)->in(1));
C->add_predicate_opaq(opq);
C->add_parse_predicate_opaq(opq);
Node* bol = new Conv2BNode(opq);
register_new_node(opq, ctrl);
register_new_node(bol, ctrl);
@ -333,22 +487,25 @@ ProjNode* PhaseIdealLoop::clone_predicate_to_unswitched_loop(ProjNode* predicate
return new_predicate_proj;
}
// Clones skeleton predicates starting at 'old_predicate_proj' by following its control inputs and rewires the control edges of in the loop from
// the old predicates to the new cloned predicates.
void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason,
ProjNode* old_predicate_proj, ProjNode* iffast_pred, ProjNode* ifslow_pred) {
// Clones Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its control inputs.
// It also rewires the control edges of data nodes with dependencies in the loop from the old predicates to the new
// cloned predicates.
void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
Deoptimization::DeoptReason reason,
IfProjNode* old_predicate_proj, IfProjNode* iffast_pred,
IfProjNode* ifslow_pred) {
assert(iffast_pred->in(0)->is_If() && ifslow_pred->in(0)->is_If(), "sanity check");
// Only need to clone range check predicates as those can be changed and duplicated by inserting pre/main/post loops
// and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the
// original predicate order.
Unique_Node_List list;
get_skeleton_predicates(old_predicate_proj, list);
get_assertion_predicates(old_predicate_proj, list);
Node_List to_process;
IfNode* iff = old_predicate_proj->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - old_predicate_proj->as_Proj()->_con);
// Process in reverse order such that 'create_new_if_for_predicate' can be used in 'clone_skeleton_predicate_for_unswitched_loops'
// and the original order is maintained.
IfProjNode* uncommon_proj = iff->proj_out(1 - old_predicate_proj->as_Proj()->_con)->as_IfProj();
// Process in reverse order such that 'create_new_if_for_predicate' can be used in
// 'clone_assertion_predicate_for_unswitched_loops' and the original order is maintained.
for (int i = list.size() - 1; i >= 0; i--) {
Node* predicate = list.at(i);
assert(predicate->in(0)->is_If(), "must be If node");
@ -356,10 +513,10 @@ void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree*
assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj(), "predicate must be a projection of an if node");
IfProjNode* predicate_proj = predicate->as_IfProj();
ProjNode* fast_proj = clone_skeleton_predicate_for_unswitched_loops(iff, predicate_proj, reason, iffast_pred);
assert(skeleton_predicate_has_opaque(fast_proj->in(0)->as_If()), "must find skeleton predicate for fast loop");
ProjNode* slow_proj = clone_skeleton_predicate_for_unswitched_loops(iff, predicate_proj, reason, ifslow_pred);
assert(skeleton_predicate_has_opaque(slow_proj->in(0)->as_If()), "must find skeleton predicate for slow loop");
IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, iffast_pred);
assert(assertion_predicate_has_loop_opaque_node(fast_proj->in(0)->as_If()), "must find Assertion Predicate for fast loop");
IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, ifslow_pred);
assert(assertion_predicate_has_loop_opaque_node(slow_proj->in(0)->as_If()), "must find Assertion Predicate for slow loop");
// Update control dependent data nodes.
for (DUIterator j = predicate->outs(); predicate->has_out(j); j++) {
@ -381,9 +538,9 @@ void PhaseIdealLoop::clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree*
}
}
// Put all skeleton predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
// is set, then the Opaque4 nodes of the skeleton predicates are put on the list instead of the projections.
void PhaseIdealLoop::get_skeleton_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) {
// Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
// is set, then the Opaque4 nodes of the Assertion Predicates are put on the list instead of the projections.
void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) {
IfNode* iff = predicate->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
@ -396,7 +553,7 @@ void PhaseIdealLoop::get_skeleton_predicates(Node* predicate, Unique_Node_List&
if (uncommon_proj->unique_ctrl_out() != rgn) {
break;
}
if (iff->in(1)->Opcode() == Op_Opaque4 && skeleton_predicate_has_opaque(iff)) {
if (iff->in(1)->Opcode() == Op_Opaque4 && assertion_predicate_has_loop_opaque_node(iff)) {
if (get_opaque) {
// Collect the predicate Opaque4 node.
list.push(iff->in(1));
@ -409,40 +566,40 @@ void PhaseIdealLoop::get_skeleton_predicates(Node* predicate, Unique_Node_List&
}
}
// Clone a skeleton predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
// predicate again).
ProjNode* PhaseIdealLoop::clone_skeleton_predicate_for_unswitched_loops(Node* iff, ProjNode* predicate,
Deoptimization::DeoptReason reason,
ProjNode* output_proj) {
Node* bol = clone_skeleton_predicate_bool(iff, nullptr, nullptr, output_proj);
ProjNode* proj = create_new_if_for_predicate(output_proj, nullptr, reason, iff->Opcode(),
IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
Deoptimization::DeoptReason reason,
IfProjNode* output_proj) {
Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, output_proj);
IfProjNode* if_proj = create_new_if_for_predicate(output_proj, nullptr, reason, iff->Opcode(),
false, predicate->is_IfTrue());
_igvn.replace_input_of(proj->in(0), 1, bol);
_igvn.replace_input_of(output_proj->in(0), 0, proj);
set_idom(output_proj->in(0), proj, dom_depth(proj));
return proj;
_igvn.replace_input_of(if_proj->in(0), 1, bol);
_igvn.replace_input_of(output_proj->in(0), 0, if_proj);
set_idom(output_proj->in(0), if_proj, dom_depth(if_proj));
return if_proj;
}
//--------------------------clone_loop_predicates-----------------------
// Clone loop predicates to cloned loops when unswitching a loop.
void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred) {
// Clone Parse Predicates to cloned loops when unswitching a loop.
void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) {
LoopNode* head = loop->_head->as_Loop();
bool clone_limit_check = !head->is_CountedLoop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
// Search original predicates
ProjNode* limit_check_proj = nullptr;
ParsePredicateSuccessProj* limit_check_proj = nullptr;
limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (limit_check_proj != nullptr) {
entry = skip_loop_predicates(entry);
entry = skip_related_predicates(entry);
}
ProjNode* profile_predicate_proj = nullptr;
ProjNode* predicate_proj = nullptr;
ParsePredicateSuccessProj* profile_predicate_proj = nullptr;
ParsePredicateSuccessProj* predicate_proj = nullptr;
if (UseProfiledLoopPredicate) {
profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (profile_predicate_proj != nullptr) {
entry = skip_loop_predicates(entry);
entry = skip_related_predicates(entry);
}
}
if (UseLoopPredicate) {
@ -450,36 +607,39 @@ void PhaseIdealLoop::clone_predicates_to_unswitched_loop(IdealLoopTree* loop, No
}
if (predicate_proj != nullptr) { // right pattern that can be used by loop predication
// clone predicate
iffast_pred = clone_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate,false);
ifslow_pred = clone_predicate_to_unswitched_loop(predicate_proj, ifslow_pred, Deoptimization::Reason_predicate,true);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj, iffast_pred, ifslow_pred);
iffast_pred = clone_parse_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate, false);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(predicate_proj, ifslow_pred, Deoptimization::Reason_predicate, true);
clone_assertion_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj,
iffast_pred, ifslow_pred);
check_created_predicate_for_unswitching(iffast_pred);
check_created_predicate_for_unswitching(ifslow_pred);
check_cloned_parse_predicate_for_unswitching(iffast_pred);
check_cloned_parse_predicate_for_unswitching(ifslow_pred);
}
if (profile_predicate_proj != nullptr) { // right pattern that can be used by loop predication
// clone predicate
iffast_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred,Deoptimization::Reason_profile_predicate, false);
ifslow_pred = clone_predicate_to_unswitched_loop(profile_predicate_proj, ifslow_pred,Deoptimization::Reason_profile_predicate, true);
clone_skeleton_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate, profile_predicate_proj, iffast_pred, ifslow_pred);
iffast_pred = clone_parse_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred,Deoptimization::Reason_profile_predicate, false);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(profile_predicate_proj, ifslow_pred,Deoptimization::Reason_profile_predicate, true);
clone_assertion_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate,
profile_predicate_proj, iffast_pred, ifslow_pred);
check_created_predicate_for_unswitching(iffast_pred);
check_created_predicate_for_unswitching(ifslow_pred);
check_cloned_parse_predicate_for_unswitching(iffast_pred);
check_cloned_parse_predicate_for_unswitching(ifslow_pred);
}
if (limit_check_proj != nullptr && clone_limit_check) {
// Clone loop limit check last to insert it before loop.
// Don't clone a limit check which was already finalized
// for this counted loop (only one limit check is needed).
iffast_pred = clone_predicate_to_unswitched_loop(limit_check_proj, iffast_pred,Deoptimization::Reason_loop_limit_check, false);
ifslow_pred = clone_predicate_to_unswitched_loop(limit_check_proj, ifslow_pred,Deoptimization::Reason_loop_limit_check, true);
iffast_pred = clone_parse_predicate_to_unswitched_loop(limit_check_proj, iffast_pred,Deoptimization::Reason_loop_limit_check, false);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(limit_check_proj, ifslow_pred,Deoptimization::Reason_loop_limit_check, true);
check_created_predicate_for_unswitching(iffast_pred);
check_created_predicate_for_unswitching(ifslow_pred);
check_cloned_parse_predicate_for_unswitching(iffast_pred);
check_cloned_parse_predicate_for_unswitching(ifslow_pred);
}
}
#ifndef PRODUCT
void PhaseIdealLoop::check_created_predicate_for_unswitching(const Node* new_entry) {
void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry) {
assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate");
if (TraceLoopPredicate) {
tty->print("Loop Predicate cloned: ");
@ -488,10 +648,7 @@ void PhaseIdealLoop::check_created_predicate_for_unswitching(const Node* new_ent
}
#endif
//--------------------------skip_loop_predicates------------------------------
// Skip related predicates.
Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) {
Node* PhaseIdealLoop::skip_related_predicates(Node* entry) {
IfNode* iff = entry->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
@ -506,23 +663,23 @@ Node* PhaseIdealLoop::skip_loop_predicates(Node* entry) {
return entry;
}
Node* PhaseIdealLoop::skip_all_loop_predicates(Node* entry) {
Predicates predicates(entry);
return predicates.skip_all();
Node* PhaseIdealLoop::skip_all_predicates(Node* entry) {
ParsePredicates parse_predicates(entry);
return parse_predicates.get_first_predicate();
}
//--------------------------next_predicate---------------------------------
// Find next related predicate, useful for iterating over all related predicates
ProjNode* PhaseIdealLoop::next_predicate(ProjNode* predicate) {
IfNode* iff = predicate->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - predicate->_con);
IfProjNode* PhaseIdealLoop::next_predicate(IfProjNode* predicate_proj) {
IfNode* iff = predicate_proj->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - predicate_proj->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
Node* next = iff->in(0);
if (next != nullptr && next->is_Proj() && next->in(0)->is_If()) {
if (next != nullptr && next->is_IfProj() && next->in(0)->is_If()) {
uncommon_proj = next->in(0)->as_If()->proj_out(1 - next->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() == rgn) { // lead into same region
return next->as_Proj();
return next->as_IfProj();
}
}
return nullptr;
@ -530,40 +687,38 @@ ProjNode* PhaseIdealLoop::next_predicate(ProjNode* predicate) {
//--------------------------find_predicate_insertion_point-------------------
// Find a good location to insert a predicate
ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) {
if (start_c == nullptr || !start_c->is_Proj())
ParsePredicateSuccessProj* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) {
if (start_c == nullptr || !start_c->is_IfTrue())
return nullptr;
if (start_c->as_Proj()->is_uncommon_trap_if_pattern(reason)) {
return start_c->as_Proj();
if (start_c->as_IfTrue()->is_uncommon_trap_if_pattern(reason)) {
return start_c->as_IfTrue();
}
return nullptr;
}
//--------------------------Predicates::Predicates--------------------------
// given loop entry, find all predicates above loop
PhaseIdealLoop::Predicates::Predicates(Node* entry) {
_loop_limit_check = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (_loop_limit_check != nullptr) {
entry = skip_loop_predicates(entry);
PhaseIdealLoop::ParsePredicates::ParsePredicates(Node* entry) {
_loop_limit_check_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (_loop_limit_check_predicate != nullptr) {
entry = skip_related_predicates(entry);
}
if (UseProfiledLoopPredicate) {
_profile_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (_profile_predicate != nullptr) {
entry = skip_loop_predicates(entry);
_profiled_loop_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (_profiled_loop_predicate != nullptr) {
entry = skip_related_predicates(entry);
}
}
if (UseLoopPredicate) {
_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (_predicate != nullptr) {
entry = skip_loop_predicates(entry);
_loop_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (_loop_predicate != nullptr) {
entry = skip_related_predicates(entry);
}
}
_entry_to_all_predicates = entry;
_first_predicate = entry;
}
//--------------------------find_predicate------------------------------------
// Find a predicate
Node* PhaseIdealLoop::find_predicate(Node* entry) {
Node* PhaseIdealLoop::find_parse_predicate(Node* entry) {
Node* predicate = nullptr;
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != nullptr) { // right pattern that can be used by loop predication
@ -1006,7 +1161,7 @@ BoolNode* PhaseIdealLoop::rc_predicate(IdealLoopTree *loop, Node* ctrl,
// Should loop predication look not only in the path from tail to head
// but also in branches of the loop body?
bool PhaseIdealLoop::loop_predication_should_follow_branches(IdealLoopTree *loop, ProjNode *predicate_proj, float& loop_trip_cnt) {
bool PhaseIdealLoop::loop_predication_should_follow_branches(IdealLoopTree* loop, IfProjNode* predicate_proj, float& loop_trip_cnt) {
if (!UseProfiledLoopPredicate) {
return false;
}
@ -1256,29 +1411,28 @@ void PhaseIdealLoop::loop_predication_follow_branches(Node *n, IdealLoopTree *lo
} while (stack.size() > 0);
}
bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* proj, ProjNode *predicate_proj,
CountedLoopNode *cl, ConNode* zero, Invariance& invar,
Deoptimization::DeoptReason reason) {
bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_proj,
ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl,
ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason) {
// Following are changed to nonnull when a predicate can be hoisted
ProjNode* new_predicate_proj = nullptr;
IfNode* iff = proj->in(0)->as_If();
IfProjNode* new_predicate_proj = nullptr;
IfNode* iff = if_proj->in(0)->as_If();
Node* test = iff->in(1);
if (!test->is_Bool()){ //Conv2B, ...
if (!test->is_Bool()) { //Conv2B, ...
return false;
}
BoolNode* bol = test->as_Bool();
if (invar.is_invariant(bol)) {
// Invariant test
new_predicate_proj = create_new_if_for_predicate(predicate_proj, nullptr,
new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr,
reason,
iff->Opcode());
Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool();
// Negate test if necessary
// Negate test if necessary (Parse Predicates always have IfTrue as success projection and IfFalse as uncommon trap)
bool negated = false;
if (proj->_con != predicate_proj->_con) {
if (if_proj->is_IfFalse()) {
new_predicate_bol = new BoolNode(new_predicate_bol->in(1), new_predicate_bol->_test.negate());
register_new_node(new_predicate_bol, ctrl);
negated = true;
@ -1295,7 +1449,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode*
loop->dump_head();
}
#endif
} else if (cl != nullptr && loop->is_range_check_if(iff, this, invar DEBUG_ONLY(COMMA predicate_proj))) {
} else if (cl != nullptr && loop->is_range_check_if(iff, this, invar DEBUG_ONLY(COMMA parse_predicate_proj))) {
// Range check for counted loops
const Node* cmp = bol->in(1)->as_Cmp();
Node* idx = cmp->in(1);
@ -1323,7 +1477,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode*
// Perform cloning to keep Invariance state correct since the
// late schedule will place invariant things in the loop.
Node *ctrl = predicate_proj->in(0)->as_If()->in(0);
Node* ctrl = parse_predicate_proj->in(0)->as_If()->in(0);
rng = invar.clone(rng, ctrl);
if (offset && offset != zero) {
assert(invar.is_invariant(offset), "offset must be loop invariant");
@ -1331,12 +1485,14 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode*
}
// If predicate expressions may overflow in the integer range, longs are used.
bool overflow = false;
bool negate = (proj->_con != predicate_proj->_con);
// Negate test if necessary (Parse Predicates always have IfTrue as success projection and IfFalse as uncommon trap)
const bool negate = (if_proj->is_IfFalse());
// Test the lower bound
BoolNode* lower_bound_bol = rc_predicate(loop, ctrl, scale, offset, init, limit, stride, rng, false, overflow, negate);
ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
const int if_opcode = iff->Opcode();
IfProjNode* lower_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode);
IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
_igvn.hash_delete(lower_bound_iff);
lower_bound_iff->set_req(1, lower_bound_bol);
@ -1345,7 +1501,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode*
// Test the upper bound
BoolNode* upper_bound_bol = rc_predicate(loop, lower_bound_proj, scale, offset, init, limit, stride, rng, true, overflow, negate);
ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
IfProjNode* upper_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode);
assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate");
IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
_igvn.hash_delete(upper_bound_iff);
@ -1355,8 +1511,8 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode*
// Fall through into rest of the cleanup code which will move any dependent nodes to the skeleton predicates of the
// upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later
// splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.).
new_predicate_proj = insert_initial_skeleton_predicate(iff, loop, proj, predicate_proj, upper_bound_proj, scale,
offset, init, limit, stride, rng, overflow, reason);
new_predicate_proj = add_template_assertion_predicate(iff, loop, if_proj, parse_predicate_proj, upper_bound_proj, scale,
offset, init, limit, stride, rng, overflow, reason);
#ifndef PRODUCT
if (TraceLoopOpts && !TraceLoopPredicate) {
@ -1371,37 +1527,31 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree *loop, ProjNode*
}
assert(new_predicate_proj != nullptr, "sanity");
// Success - attach condition (new_predicate_bol) to predicate if
invar.map_ctrl(proj, new_predicate_proj); // so that invariance test can be appropriate
invar.map_ctrl(if_proj, new_predicate_proj); // so that invariance test can be appropriate
// Eliminate the old If in the loop body
dominated_by( new_predicate_proj->as_IfProj(), iff, proj->_con != new_predicate_proj->_con );
dominated_by(new_predicate_proj, iff, if_proj->_con != new_predicate_proj->_con );
C->set_major_progress();
return true;
}
// After pre/main/post loops are created, we'll put a copy of some
// range checks between the pre and main loop to validate the value
// of the main loop induction variable. Make a copy of the predicates
// here with an opaque node as a place holder for the value (will be
// updated by PhaseIdealLoop::clone_skeleton_predicate()).
ProjNode* PhaseIdealLoop::insert_initial_skeleton_predicate(IfNode* iff, IdealLoopTree *loop,
ProjNode* proj, ProjNode *predicate_proj,
ProjNode* upper_bound_proj,
int scale, Node* offset,
Node* init, Node* limit, jint stride,
Node* rng, bool &overflow,
Deoptimization::DeoptReason reason) {
// Each newly created Hoisted Predicate is accompanied by two Template Assertion Predicates. Later, we initialize them
// by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub loops
// are removed properly.
IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
IfProjNode* predicate_proj, IfProjNode* upper_bound_proj,
int scale, Node* offset, Node* init, Node* limit, jint stride,
Node* rng, bool& overflow, Deoptimization::DeoptReason reason) {
// First predicate for the initial value on first loop iteration
Node* opaque_init = new OpaqueLoopInitNode(C, init);
register_new_node(opaque_init, upper_bound_proj);
bool negate = (proj->_con != predicate_proj->_con);
bool negate = (if_proj->_con != predicate_proj->_con);
BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng, (stride > 0) != (scale > 0), overflow, negate);
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over
C->add_skeleton_predicate_opaq(opaque_bol);
C->add_template_assertion_predicate_opaq(opaque_bol);
register_new_node(opaque_bol, upper_bound_proj);
ProjNode* new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
IfProjNode* new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(opaque_init->outcnt() > 0, "should be used");
@ -1421,12 +1571,12 @@ ProjNode* PhaseIdealLoop::insert_initial_skeleton_predicate(IfNode* iff, IdealLo
bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0), overflow, negate);
opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1));
C->add_skeleton_predicate_opaq(opaque_bol);
C->add_template_assertion_predicate_opaq(opaque_bol);
register_new_node(opaque_bol, new_proj);
new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(max_value->outcnt() > 0, "should be used");
assert(skeleton_predicate_has_opaque(new_proj->in(0)->as_If()), "unexpected");
assert(assertion_predicate_has_loop_opaque_node(new_proj->in(0)->as_If()), "unexpected");
return new_proj;
}
@ -1463,18 +1613,18 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
}
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
ProjNode *loop_limit_proj = nullptr;
ProjNode *predicate_proj = nullptr;
ProjNode *profile_predicate_proj = nullptr;
ParsePredicateSuccessProj* loop_limit_proj = nullptr;
ParsePredicateSuccessProj* predicate_proj = nullptr;
ParsePredicateSuccessProj* profile_predicate_proj = nullptr;
// Loop limit check predicate should be near the loop.
loop_limit_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (loop_limit_proj != nullptr) {
entry = skip_loop_predicates(loop_limit_proj);
entry = skip_related_predicates(loop_limit_proj);
}
bool has_profile_predicates = false;
profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (profile_predicate_proj != nullptr) {
Node* n = skip_loop_predicates(entry);
Node* n = skip_related_predicates(entry);
// Check if predicates were already added to the profile predicate
// block
if (n != entry->in(0)->in(0) || n->outcnt() != 1) {
@ -1533,10 +1683,10 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
while (if_proj_list.size() > 0) {
Node* n = if_proj_list.pop();
ProjNode* proj = n->as_Proj();
IfNode* iff = proj->in(0)->as_If();
IfProjNode* if_proj = n->as_IfProj();
IfNode* iff = if_proj->in(0)->as_If();
CallStaticJavaNode* call = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
if (call == nullptr) {
if (loop->is_loop_exit(iff)) {
// stop processing the remaining projs in the list because the execution of them
@ -1559,7 +1709,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
}
if (predicate_proj != nullptr) {
hoisted = loop_predication_impl_helper(loop, proj, predicate_proj, cl, zero, invar, Deoptimization::Reason_predicate) | hoisted;
hoisted = loop_predication_impl_helper(loop, if_proj, predicate_proj, cl, zero, invar, Deoptimization::Reason_predicate) | hoisted;
}
} // end while
}
@ -1570,11 +1720,11 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
// Some projections were skipped by regular predicates because of
// an early loop exit. Try them with profile data.
while (if_proj_list.size() > 0) {
Node* proj = if_proj_list.pop();
float f = pf.to(proj);
if (proj->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
Node* if_proj = if_proj_list.pop();
float f = pf.to(if_proj);
if (if_proj->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
f * loop_trip_cnt >= 1) {
hoisted = loop_predication_impl_helper(loop, proj->as_Proj(), profile_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted;
hoisted = loop_predication_impl_helper(loop, if_proj->as_IfProj(), profile_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted;
}
}
@ -1588,8 +1738,8 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
}
for (uint i = 0; i < if_proj_list_freq.size(); i++) {
ProjNode* proj = if_proj_list_freq.at(i)->as_Proj();
hoisted = loop_predication_impl_helper(loop, proj, profile_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted;
IfProjNode* if_proj = if_proj_list_freq.at(i)->as_IfProj();
hoisted = loop_predication_impl_helper(loop, if_proj, profile_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted;
}
}

305
src/hotspot/share/opto/loopTransform.cpp
View File

@ -560,7 +560,7 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne
// stmt1
// |
// v
// loop predicate
// predicates
// |
// v
// loop<----+
@ -585,7 +585,7 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne
// stmt1
// |
// v
// loop predicate
// predicates
// / \
// clone / \ orig
// / \
@ -616,7 +616,7 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne
// stmt1
// |
// v
// loop predicate
// predicates
// /
// /
// clone / orig
@ -653,7 +653,7 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne
// stmt1
// |
// v
// loop predicate
// predicates
// |
// v
// stmt2 clone
@ -666,7 +666,7 @@ void PhaseIdealLoop::peeled_dom_test_elim(IdealLoopTree* loop, Node_List& old_ne
// false true
// | |
// | v
// | initialized skeleton predicates
// | Initialized Assertion Predicates
// | |
// | v
// | loop<----+
@ -771,21 +771,21 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {
}
}
// Step 5: skeleton_predicates instantiation
// Step 5: Assertion Predicates initialization
if (counted_loop && UseLoopPredicate) {
CountedLoopNode *cl_head = head->as_CountedLoop();
Node* init = cl_head->init_trip();
Node* stride = cl_head->stride();
IdealLoopTree* outer_loop = get_loop(outer_loop_head);
Predicates predicates(new_head->in(LoopNode::EntryControl));
initialize_skeleton_predicates_for_peeled_loop(predicates.predicate(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
initialize_skeleton_predicates_for_peeled_loop(predicates.profile_predicate(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
ParsePredicates parse_predicates(new_head->in(LoopNode::EntryControl));
initialize_assertion_predicates_for_peeled_loop(parse_predicates.loop_predicate(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
initialize_assertion_predicates_for_peeled_loop(parse_predicates.profiled_loop_predicate(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
}
// Now force out all loop-invariant dominating tests. The optimizer
@ -1302,19 +1302,17 @@ void PhaseIdealLoop::ensure_zero_trip_guard_proj(Node* node, bool is_main_loop)
}
#endif
// Make a copy of the skeleton range check predicates before the main
// loop and set the initial value of loop as input. After unrolling,
// the range of values for the induction variable in the main loop can
// fall outside the allowed range of values by the array access (main
// loop is never executed). When that happens, range check
// CastII/ConvI2L nodes cause some data paths to die. For consistency,
// the control paths must die too but the range checks were removed by
// predication. The range checks that we add here guarantee that they do.
void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride,
IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post,
const uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main,
Node* zero_trip_guard_proj_post, const Node_List &old_new) {
// Make two copies of each Template Assertion Predicate before the pre-loop and add them to the main-loop. One remains
// a template while the other one is initialized with the initial value of the loop induction variable. The Initialized
// Assertion Predicates ensures that the main-loop is removed if some type ranges of Cast or Convert nodes become
// impossible and are replaced by top (i.e. a sign that the main-loop is dead).
void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride,
IdealLoopTree* outer_loop, LoopNode* outer_main_head,
const uint dd_main_head, const uint idx_before_pre_post,
const uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main,
Node* zero_trip_guard_proj_post,
const Node_List &old_new) {
if (predicate != nullptr) {
#ifdef ASSERT
ensure_zero_trip_guard_proj(zero_trip_guard_proj_main, true);
@ -1339,20 +1337,20 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
if (uncommon_proj->unique_ctrl_out() != rgn)
break;
if (iff->in(1)->Opcode() == Op_Opaque4) {
assert(skeleton_predicate_has_opaque(iff), "unexpected");
// Clone the skeleton predicate twice and initialize one with the initial
assert(assertion_predicate_has_loop_opaque_node(iff), "unexpected");
// Clone the Assertion Predicate twice and initialize one with the initial
// value of the loop induction variable. Leave the other predicate
// to be initialized when increasing the stride during loop unrolling.
prev_proj = clone_skeleton_predicate_and_initialize(iff, opaque_init, nullptr, predicate, uncommon_proj,
current_proj, outer_loop, prev_proj);
assert(skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");
prev_proj = clone_assertion_predicate_and_initialize(iff, opaque_init, nullptr, predicate, uncommon_proj,
current_proj, outer_loop, prev_proj);
assert(assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), "");
prev_proj = clone_skeleton_predicate_and_initialize(iff, init, stride, predicate, uncommon_proj,
current_proj, outer_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "");
prev_proj = clone_assertion_predicate_and_initialize(iff, init, stride, predicate, uncommon_proj,
current_proj, outer_loop, prev_proj);
assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), "");
// Rewire any control inputs from the cloned skeleton predicates down to the main and post loop for data nodes that are part of the
// main loop (and were cloned to the pre and post loop).
// Rewire any control inputs from the cloned Assertion Predicates down to the main and post loop for data nodes
// that are part of the main loop (and were cloned to the pre and post loop).
for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
Node* loop_node = predicate->out(i);
Node* pre_loop_node = old_new[loop_node->_idx];
@ -1373,7 +1371,7 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
}
}
// Remove the skeleton predicate from the pre-loop
// Remove the Assertion Predicate from the pre-loop
_igvn.replace_input_of(iff, 1, _igvn.intcon(1));
}
predicate = predicate->in(0)->in(0);
@ -1383,7 +1381,9 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop_helper(Node* predicat
}
}
static bool skeleton_follow_inputs(Node* n) {
// Is 'n' a node that can be found on the input chain of a Template Assertion Predicate bool (i.e. between a Template
// Assertion Predicate If node and the OpaqueLoop* nodes)?
static bool is_part_of_template_assertion_predicate_bool(Node* n) {
int op = n->Opcode();
return (n->is_Bool() ||
n->is_Cmp() ||
@ -1406,7 +1406,7 @@ bool PhaseIdealLoop::subgraph_has_opaque(Node* n) {
if (n->Opcode() == Op_OpaqueLoopInit || n->Opcode() == Op_OpaqueLoopStride) {
return true;
}
if (!skeleton_follow_inputs(n)) {
if (!is_part_of_template_assertion_predicate_bool(n)) {
return false;
}
uint init;
@ -1415,8 +1415,7 @@ bool PhaseIdealLoop::subgraph_has_opaque(Node* n) {
return init != 0 || stride != 0;
}
bool PhaseIdealLoop::skeleton_predicate_has_opaque(IfNode* iff) {
bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) {
uint init;
uint stride;
count_opaque_loop_nodes(iff->in(1)->in(1), init, stride);
@ -1459,7 +1458,7 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride)
wq.push(n);
for (uint i = 0; i < wq.size(); i++) {
Node* n = wq.at(i);
if (skeleton_follow_inputs(n)) {
if (is_part_of_template_assertion_predicate_bool(n)) {
for (uint j = 1; j < n->req(); j++) {
Node* m = n->in(j);
if (m != nullptr) {
@ -1476,12 +1475,15 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride)
}
}
// Clone the skeleton predicate bool for a main or unswitched loop:
// Main loop: Set new_init and new_stride nodes as new inputs.
// Unswitched loop: new_init and new_stride are both null. Clone OpaqueLoopInit and OpaqueLoopStride instead.
Node* PhaseIdealLoop::clone_skeleton_predicate_bool(Node* iff, Node* new_init, Node* new_stride, Node* control) {
// Create a new Bool node from the provided Template Assertion Predicate.
// Unswitched loop: new_init and new_stride are both null. Clone OpaqueLoopInit and OpaqueLoopStride.
// Otherwise: Replace found OpaqueLoop* nodes with new_init and new_stride, respectively.
Node* PhaseIdealLoop::create_bool_from_template_assertion_predicate(Node* template_assertion_predicate, Node* new_init,
Node* new_stride, Node* control) {
Node_Stack to_clone(2);
to_clone.push(iff->in(1), 1);
Node* opaque4 = template_assertion_predicate->in(1);
assert(opaque4->Opcode() == Op_Opaque4, "must be Opaque4");
to_clone.push(opaque4, 1);
uint current = C->unique();
Node* result = nullptr;
bool is_unswitched_loop = new_init == nullptr && new_stride == nullptr;
@ -1495,7 +1497,7 @@ Node* PhaseIdealLoop::clone_skeleton_predicate_bool(Node* iff, Node* new_init, N
Node* n = to_clone.node();
uint i = to_clone.index();
Node* m = n->in(i);
if (skeleton_follow_inputs(m)) {
if (is_part_of_template_assertion_predicate_bool(m)) {
to_clone.push(m, 1);
continue;
}
@ -1553,11 +1555,12 @@ Node* PhaseIdealLoop::clone_skeleton_predicate_bool(Node* iff, Node* new_init, N
return result;
}
// Clone a skeleton predicate for the main loop. new_init and new_stride are set as new inputs. Since the predicates cannot fail at runtime,
// Halt nodes are inserted instead of uncommon traps.
Node* PhaseIdealLoop::clone_skeleton_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj,
Node* control, IdealLoopTree* outer_loop, Node* input_proj) {
Node* result = clone_skeleton_predicate_bool(iff, new_init, new_stride, control);
// Clone an Assertion Predicate for the main loop. new_init and new_stride are set as new inputs. Since the predicates
// cannot fail at runtime, Halt nodes are inserted instead of uncommon traps.
Node* PhaseIdealLoop::clone_assertion_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate,
Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop,
Node* input_proj) {
Node* result = create_bool_from_template_assertion_predicate(iff, new_init, new_stride, control);
Node* proj = predicate->clone();
Node* other_proj = uncommon_proj->clone();
Node* new_iff = iff->clone();
@ -1566,7 +1569,7 @@ Node* PhaseIdealLoop::clone_skeleton_predicate_and_initialize(Node* iff, Node* n
other_proj->set_req(0, new_iff);
Node* frame = new ParmNode(C->start(), TypeFunc::FramePtr);
register_new_node(frame, C->start());
// It's impossible for the predicate to fail at runtime. Use an Halt node.
// It's impossible for the predicate to fail at runtime. Use a Halt node.
Node* halt = new HaltNode(other_proj, frame, "duplicated predicate failed which is impossible");
_igvn.add_input_to(C->root(), halt);
new_iff->set_req(0, input_proj);
@ -1578,32 +1581,34 @@ Node* PhaseIdealLoop::clone_skeleton_predicate_and_initialize(Node* iff, Node* n
return proj;
}
void PhaseIdealLoop::copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride,
IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post,
const uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main,
Node* zero_trip_guard_proj_post, const Node_List &old_new) {
void PhaseIdealLoop::copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride,
IdealLoopTree* outer_loop, LoopNode* outer_main_head,
const uint dd_main_head, const uint idx_before_pre_post,
const uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main,
Node* zero_trip_guard_proj_post,
const Node_List &old_new) {
if (UseLoopPredicate) {
Node* entry = pre_head->in(LoopNode::EntryControl);
Node* predicate = nullptr;
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != nullptr) {
entry = skip_loop_predicates(entry);
entry = skip_related_predicates(entry);
}
Node* profile_predicate = nullptr;
if (UseProfiledLoopPredicate) {
profile_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (profile_predicate != nullptr) {
entry = skip_loop_predicates(entry);
entry = skip_related_predicates(entry);
}
}
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
copy_skeleton_predicates_to_main_loop_helper(predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
zero_trip_guard_proj_post, old_new);
copy_skeleton_predicates_to_main_loop_helper(profile_predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
zero_trip_guard_proj_post, old_new);
copy_assertion_predicates_to_main_loop_helper(predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
zero_trip_guard_proj_post, old_new);
copy_assertion_predicates_to_main_loop_helper(profile_predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
zero_trip_guard_proj_post, old_new);
}
}
@ -1753,9 +1758,10 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n
Node* castii = cast_incr_before_loop(pre_incr, min_taken, main_head);
assert(castii != nullptr, "no castII inserted");
assert(post_head->in(1)->is_IfProj(), "must be zero-trip guard If node projection of the post loop");
copy_skeleton_predicates_to_main_loop(pre_head, castii, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, min_taken, post_head->in(1), old_new);
copy_skeleton_predicates_to_post_loop(outer_main_head, post_head, post_incr, stride);
copy_assertion_predicates_to_main_loop(pre_head, castii, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, min_taken, post_head->in(1),
old_new);
copy_assertion_predicates_to_post_loop(outer_main_head, post_head, post_incr, stride);
// Step B4: Shorten the pre-loop to run only 1 iteration (for now).
// RCE and alignment may change this later.
@ -1878,7 +1884,7 @@ void PhaseIdealLoop::insert_vector_post_loop(IdealLoopTree *loop, Node_List &old
// In this case we throw away the result as we are not using it to connect anything else.
CountedLoopNode *post_head = nullptr;
insert_post_loop(loop, old_new, main_head, main_end, incr, limit, post_head);
copy_skeleton_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, incr, main_head->stride());
copy_assertion_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, incr, main_head->stride());
// It's difficult to be precise about the trip-counts
// for post loops. They are usually very short,
@ -1925,7 +1931,7 @@ void PhaseIdealLoop::insert_scalar_rced_post_loop(IdealLoopTree *loop, Node_List
// In this case we throw away the result as we are not using it to connect anything else.
CountedLoopNode *post_head = nullptr;
insert_post_loop(loop, old_new, main_head, main_end, incr, limit, post_head);
copy_skeleton_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, incr, main_head->stride());
copy_assertion_predicates_to_post_loop(main_head->skip_strip_mined(), post_head, incr, main_head->stride());
// It's difficult to be precise about the trip-counts
// for post loops. They are usually very short,
@ -2042,14 +2048,16 @@ bool IdealLoopTree::is_invariant(Node* n) const {
return !is_member(_phase->get_loop(n_c));
}
void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con) {
// Search the Assertion Predicates added by loop predication and/or range check elimination and update them according
// to the new stride.
void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init,
const int stride_con) {
if (init->Opcode() == Op_CastII) {
// skip over the cast added by PhaseIdealLoop::cast_incr_before_loop() when pre/post/main loops are created because
// it can get in the way of type propagation
assert(((CastIINode*)init)->carry_dependency() && loop_head->skip_predicates() == init->in(0), "casted iv phi from pre loop expected");
init = init->in(1);
}
// Search for skeleton predicates and update them according to the new stride
Node* entry = ctrl;
Node* prev_proj = ctrl;
LoopNode* outer_loop_head = loop_head->skip_strip_mined();
@ -2068,18 +2076,20 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo
break;
}
if (iff->in(1)->Opcode() == Op_Opaque4) {
// Look for predicate with an Opaque1 node that can be used as a template
if (!skeleton_predicate_has_opaque(iff)) {
// No Opaque1 node? It's either the check for the first value
// of the first iteration or the check for the last value of
// the first iteration of an unrolled loop. We can't
// tell. Kill it in any case.
if (!assertion_predicate_has_loop_opaque_node(iff)) {
// No OpaqueLoop* node? Then it's one of the two Initialized Assertion Predicates:
// - For the initial access a[init]
// - For the last access a[init+old_stride-orig_stride]
// We could keep the one for the initial access but we do not know which one we currently have here. Just kill both.
// We will create new Initialized Assertion Predicates from the Template Assertion Predicates below:
// - For the initial access a[init] (same as before)
// - For the last access a[init+new_stride-orig_stride] (with the new unroll stride)
_igvn.replace_input_of(iff, 1, iff->in(1)->in(2));
} else {
// Add back predicates updated for the new stride.
prev_proj = clone_skeleton_predicate_and_initialize(iff, init, max_value, entry, proj, ctrl, outer_loop,
prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
// Template Assertion Predicate: Clone it to create initialized version with new stride.
prev_proj = clone_assertion_predicate_and_initialize(iff, init, max_value, entry, proj, ctrl, outer_loop,
prev_proj);
assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), "unexpected");
}
}
entry = entry->in(0)->in(0);
@ -2090,9 +2100,10 @@ void PhaseIdealLoop::update_main_loop_skeleton_predicates(Node* ctrl, CountedLoo
}
}
void PhaseIdealLoop::copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* init, Node* stride) {
// Go over the skeleton predicates of the main loop and make a copy for the post loop with its initial iv value and
// stride as inputs.
// Go over the Assertion Predicates of the main loop and make a copy for the post loop with its initial iv value and
// stride as inputs.
void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head,
Node* init, Node* stride) {
Node* post_loop_entry = post_loop_head->in(LoopNode::EntryControl);
Node* main_loop_entry = main_loop_head->in(LoopNode::EntryControl);
IdealLoopTree* post_loop = get_loop(post_loop_head);
@ -2105,10 +2116,10 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_h
if (proj->unique_ctrl_out()->Opcode() != Op_Halt) {
break;
}
if (iff->in(1)->Opcode() == Op_Opaque4 && skeleton_predicate_has_opaque(iff)) {
prev_proj = clone_skeleton_predicate_and_initialize(iff, init, stride, ctrl, proj, post_loop_entry,
post_loop, prev_proj);
assert(!skeleton_predicate_has_opaque(prev_proj->in(0)->as_If()), "unexpected");
if (iff->in(1)->Opcode() == Op_Opaque4 && assertion_predicate_has_loop_opaque_node(iff)) {
prev_proj = clone_assertion_predicate_and_initialize(iff, init, stride, ctrl, proj, post_loop_entry,
post_loop, prev_proj);
assert(!assertion_predicate_has_loop_opaque_node(prev_proj->in(0)->as_If()), "unexpected");
}
ctrl = ctrl->in(0)->in(0);
}
@ -2118,32 +2129,30 @@ void PhaseIdealLoop::copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_h
}
}
void PhaseIdealLoop::initialize_skeleton_predicates_for_peeled_loop(ProjNode* predicate,
LoopNode* outer_loop_head,
int dd_outer_loop_head,
Node* init,
Node* stride,
IdealLoopTree* outer_loop,
const uint idx_before_clone,
const Node_List &old_new) {
if (predicate == nullptr) {
void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(IfProjNode* predicate_proj, LoopNode* outer_loop_head,
const int dd_outer_loop_head, Node* init,
Node* stride, IdealLoopTree* outer_loop,
const uint idx_before_clone,
const Node_List &old_new) {
if (predicate_proj == nullptr) {
return;
}
Node* control = outer_loop_head->in(LoopNode::EntryControl);
Node* input_proj = control;
predicate = next_predicate(predicate);
while (predicate != nullptr) {
IfNode* iff = predicate->in(0)->as_If();
predicate_proj = next_predicate(predicate_proj);
while (predicate_proj != nullptr) {
IfNode* iff = predicate_proj->in(0)->as_If();
if (iff->in(1)->Opcode() == Op_Opaque4) {
assert(skeleton_predicate_has_opaque(iff), "unexpected");
ProjNode* uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
input_proj = clone_skeleton_predicate_and_initialize(iff, init, stride, predicate, uncommon_proj, control, outer_loop, input_proj);
assert(assertion_predicate_has_loop_opaque_node(iff), "unexpected");
ProjNode* uncommon_proj = iff->proj_out(1 - predicate_proj->as_Proj()->_con);
input_proj = clone_assertion_predicate_and_initialize(iff, init, stride, predicate_proj, uncommon_proj, control,
outer_loop, input_proj);
// Rewire any control inputs from the old skeleton predicates above the peeled iteration down to the initialized
// skeleton predicates above the peeled loop.
for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
Node* dependent = predicate->out(i);
// Rewire any control inputs from the old Assertion Predicates above the peeled iteration down to the initialized
// Assertion Predicates above the peeled loop.
for (DUIterator i = predicate_proj->outs(); predicate_proj->has_out(i); i++) {
Node* dependent = predicate_proj->out(i);
Node* new_node = old_new[dependent->_idx];
if (!dependent->is_CFG() &&
@ -2151,13 +2160,13 @@ void PhaseIdealLoop::initialize_skeleton_predicates_for_peeled_loop(ProjNode* pr
new_node != nullptr && // cloned
new_node->_idx >= idx_before_clone) { // for peeling
// The old nodes from the peeled loop still point to the predicate above the peeled loop.
// We need to rewire the dependencies to the newly initialized skeleton predicates.
// We need to rewire the dependencies to the newly Initialized Assertion Predicates.
_igvn.replace_input_of(dependent, 0, input_proj);
--i; // correct for just deleted predicate->out(i)
}
}
}
predicate = next_predicate(predicate);
predicate_proj = next_predicate(predicate_proj);
}
_igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, input_proj);
@ -2227,7 +2236,7 @@ void PhaseIdealLoop::do_unroll(IdealLoopTree *loop, Node_List &old_new, bool adj
assert(old_trip_count > 1 && (!adjust_min_trip || stride_p <=
MIN2<int>(max_jint / 2 - 2, MAX2(1<<3, Matcher::max_vector_size(T_BYTE)) * loop_head->unrolled_count())), "sanity");
update_main_loop_skeleton_predicates(ctrl, loop_head, init, stride_con);
update_main_loop_assertion_predicates(ctrl, loop_head, init, stride_con);
// Adjust loop limit to keep valid iterations number after unroll.
// Use (limit - stride) instead of (((limit - init)/stride) & (-2))*stride
@ -2852,20 +2861,22 @@ bool PhaseIdealLoop::is_scaled_iv_plus_extra_offset(Node* exp1, Node* offset3, N
// Same as PhaseIdealLoop::duplicate_predicates() but for range checks
// eliminated by iteration splitting.
Node* PhaseIdealLoop::add_range_check_predicate(IdealLoopTree* loop, CountedLoopNode* cl,
Node* predicate_proj, int scale_con, Node* offset,
Node* limit, jint stride_con, Node* value) {
Node* PhaseIdealLoop::add_range_check_elimination_assertion_predicate(IdealLoopTree* loop,
Node* ctrl, const int scale_con,
Node* offset, Node* limit, jint stride_con,
Node* value) {
bool overflow = false;
BoolNode* bol = rc_predicate(loop, predicate_proj, scale_con, offset, value, nullptr, stride_con, limit, (stride_con > 0) != (scale_con > 0), overflow, false);
BoolNode* bol = rc_predicate(loop, ctrl, scale_con, offset, value, nullptr, stride_con,
limit, (stride_con > 0) != (scale_con > 0), overflow, false);
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1));
register_new_node(opaque_bol, predicate_proj);
register_new_node(opaque_bol, ctrl);
IfNode* new_iff = nullptr;
if (overflow) {
new_iff = new IfNode(predicate_proj, opaque_bol, PROB_MAX, COUNT_UNKNOWN);
new_iff = new IfNode(ctrl, opaque_bol, PROB_MAX, COUNT_UNKNOWN);
} else {
new_iff = new RangeCheckNode(predicate_proj, opaque_bol, PROB_MAX, COUNT_UNKNOWN);
new_iff = new RangeCheckNode(ctrl, opaque_bol, PROB_MAX, COUNT_UNKNOWN);
}
register_control(new_iff, loop->_parent, predicate_proj);
register_control(new_iff, loop->_parent, ctrl);
Node* iffalse = new IfFalseNode(new_iff);
register_control(iffalse, _ltree_root, new_iff);
ProjNode* iftrue = new IfTrueNode(new_iff);
@ -2956,8 +2967,8 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
set_ctrl(one, C->root());
set_ctrl(mini, C->root());
Node* predicate_proj = cl->skip_strip_mined()->in(LoopNode::EntryControl);
assert(predicate_proj->is_Proj() && predicate_proj->in(0)->is_If(), "if projection only");
Node* loop_entry = cl->skip_strip_mined()->in(LoopNode::EntryControl);
assert(loop_entry->is_Proj() && loop_entry->in(0)->is_If(), "if projection only");
// Check loop body for tests of trip-counter plus loop-invariant vs loop-variant.
for (uint i = 0; i < loop->_body.size(); i++) {
@ -3054,27 +3065,31 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
add_constraint(stride_con, lscale_con, offset, zero, limit, pre_ctrl, &pre_limit, &main_limit);
Node* init = cl->init_trip();
Node* opaque_init = new OpaqueLoopInitNode(C, init);
register_new_node(opaque_init, predicate_proj);
register_new_node(opaque_init, loop_entry);
// predicate on first value of first iteration
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, int_offset, int_limit, stride_con, init);
assert(!skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
// Initialized Assertion Predicate for the value of the initial main-loop.
loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset,
int_limit, stride_con, init);
assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
// template predicate so it can be updated on next unrolling
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, int_offset, int_limit, stride_con, opaque_init);
assert(skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
// Add two Template Assertion Predicates to create new Initialized Assertion Predicates from when either
// unrolling or splitting this main-loop further.
loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset,
int_limit, stride_con, opaque_init);
assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
Node* opaque_stride = new OpaqueLoopStrideNode(C, cl->stride());
register_new_node(opaque_stride, predicate_proj);
register_new_node(opaque_stride, loop_entry);
Node* max_value = new SubINode(opaque_stride, cl->stride());
register_new_node(max_value, predicate_proj);
register_new_node(max_value, loop_entry);
max_value = new AddINode(opaque_init, max_value);
register_new_node(max_value, predicate_proj);
register_new_node(max_value, loop_entry);
// init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi
max_value = new CastIINode(max_value, loop->_head->as_CountedLoop()->phi()->bottom_type());
register_new_node(max_value, predicate_proj);
predicate_proj = add_range_check_predicate(loop, cl, predicate_proj, scale_con, int_offset, int_limit, stride_con, max_value);
assert(skeleton_predicate_has_opaque(predicate_proj->in(0)->as_If()), "unexpected");
register_new_node(max_value, loop_entry);
loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset,
int_limit, stride_con, max_value);
assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
} else {
if (PrintOpto) {
@ -3136,9 +3151,9 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
}
} // End of is IF
}
if (predicate_proj != cl->skip_strip_mined()->in(LoopNode::EntryControl)) {
_igvn.replace_input_of(cl->skip_strip_mined(), LoopNode::EntryControl, predicate_proj);
set_idom(cl->skip_strip_mined(), predicate_proj, dom_depth(cl->skip_strip_mined()));
if (loop_entry != cl->skip_strip_mined()->in(LoopNode::EntryControl)) {
_igvn.replace_input_of(cl->skip_strip_mined(), LoopNode::EntryControl, loop_entry);
set_idom(cl->skip_strip_mined(), loop_entry, dom_depth(cl->skip_strip_mined()));
}
// Update loop limits
@ -3519,11 +3534,11 @@ bool IdealLoopTree::do_remove_empty_loop(PhaseIdealLoop *phase) {
}
if (needs_guard) {
// Check for an obvious zero trip guard.
Node* inctrl = PhaseIdealLoop::skip_all_loop_predicates(cl->skip_predicates());
if (inctrl->Opcode() == Op_IfTrue || inctrl->Opcode() == Op_IfFalse) {
bool maybe_swapped = (inctrl->Opcode() == Op_IfFalse);
Node* in_ctrl = PhaseIdealLoop::skip_all_predicates(cl->skip_predicates());
if (in_ctrl->Opcode() == Op_IfTrue || in_ctrl->Opcode() == Op_IfFalse) {
bool maybe_swapped = (in_ctrl->Opcode() == Op_IfFalse);
// The test should look like just the backedge of a CountedLoop
Node* iff = inctrl->in(0);
Node* iff = in_ctrl->in(0);
if (iff->is_If()) {
Node* bol = iff->in(1);
if (bol->is_Bool()) {

44
src/hotspot/share/opto/loopUnswitch.cpp
View File

@ -34,14 +34,14 @@
//
// orig: transformed:
// if (invariant-test) then
// predicate predicate
// predicates predicates
// loop loop
// stmt1 stmt1
// if (invariant-test) then stmt2
// stmt2 stmt4
// else endloop
// stmt3 else
// endif predicate [clone]
// endif predicates [clone]
// stmt4 loop [clone]
// endloop stmt1 [clone]
// stmt3
@ -125,9 +125,9 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) {
|| (UseLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_predicate) != nullptr)) {
assert(entry->is_IfProj(), "sanity - must be ifProj since there is at least one predicate");
if (entry->outcnt() > 1) {
// Bailout if there are loop predicates from which there are additional control dependencies (i.e. from
// loop entry 'entry') to previously partially peeled statements since this case is not handled and can lead
// to wrong execution. Remove this bailout, once this is fixed.
// Bailout if there are predicates from which there are additional control dependencies (i.e. from loop
// entry 'entry') to previously partially peeled statements since this case is not handled and can lead
// to a wrong execution. Remove this bailout, once this is fixed.
return;
}
}
@ -154,29 +154,29 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) {
#ifdef ASSERT
assert(proj_true->is_IfTrue(), "must be true projection");
entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
Node* predicate = find_predicate(entry);
if (predicate == nullptr) {
// No empty predicate
Node* parse_predicate = find_parse_predicate(entry);
if (parse_predicate == nullptr) {
// No Parse Predicate.
Node* uniqc = proj_true->unique_ctrl_out();
assert((uniqc == head && !head->is_strip_mined()) || (uniqc == head->in(LoopNode::EntryControl)
&& head->is_strip_mined()), "must hold by construction if no predicates");
} else {
// There is at least one empty predicate. When calling 'skip_loop_predicates' on each found empty predicate,
// There is at least one Parse Predicate. When calling 'skip_related_predicates' on each found Parse Predicate,
// we should end up at 'proj_true'.
Node* proj_before_first_empty_predicate = skip_loop_predicates(entry);
Node* proj_before_first_parse_predicate = skip_related_predicates(entry);
if (UseProfiledLoopPredicate) {
predicate = find_predicate(proj_before_first_empty_predicate);
if (predicate != nullptr) {
proj_before_first_empty_predicate = skip_loop_predicates(predicate);
parse_predicate = find_parse_predicate(proj_before_first_parse_predicate);
if (parse_predicate != nullptr) {
proj_before_first_parse_predicate = skip_related_predicates(parse_predicate);
}
}
if (UseLoopPredicate) {
predicate = find_predicate(proj_before_first_empty_predicate);
if (predicate != nullptr) {
proj_before_first_empty_predicate = skip_loop_predicates(predicate);
parse_predicate = find_parse_predicate(proj_before_first_parse_predicate);
if (parse_predicate != nullptr) {
proj_before_first_parse_predicate = skip_related_predicates(parse_predicate);
}
}
assert(proj_true == proj_before_first_empty_predicate, "must hold by construction if at least one predicate");
assert(proj_true == proj_before_first_parse_predicate, "must hold by construction if at least one predicate");
}
#endif
// Increment unswitch count
@ -258,9 +258,9 @@ IfNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
IfNode* iff = (unswitch_iff->Opcode() == Op_RangeCheck) ? new RangeCheckNode(entry, bol, unswitch_iff->_prob, unswitch_iff->_fcnt) :
new IfNode(entry, bol, unswitch_iff->_prob, unswitch_iff->_fcnt);
register_node(iff, outer_loop, entry, dom_depth(entry));
ProjNode* iffast = new IfTrueNode(iff);
IfProjNode* iffast = new IfTrueNode(iff);
register_node(iffast, outer_loop, iff, dom_depth(iff));
ProjNode* ifslow = new IfFalseNode(iff);
IfProjNode* ifslow = new IfFalseNode(iff);
register_node(ifslow, outer_loop, iff, dom_depth(iff));
// Clone the loop body. The clone becomes the slow loop. The
@ -270,9 +270,9 @@ IfNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
assert(old_new[head->_idx]->is_Loop(), "" );
// Fast (true) and Slow (false) control
ProjNode* iffast_pred = iffast;
ProjNode* ifslow_pred = ifslow;
clone_predicates_to_unswitched_loop(loop, old_new, iffast_pred, ifslow_pred);
IfProjNode* iffast_pred = iffast;
IfProjNode* ifslow_pred = ifslow;
clone_parse_and_assertion_predicates_to_unswitched_loop(loop, old_new, iffast_pred, ifslow_pred);
Node* l = head->skip_strip_mined();
_igvn.replace_input_of(l, LoopNode::EntryControl, iffast_pred);

67
src/hotspot/share/opto/loopnode.cpp
View File

@ -341,8 +341,9 @@ IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(BoolNode *test, Nod
return outer_ilt;
}
void PhaseIdealLoop::insert_loop_limit_check(ProjNode* limit_check_proj, Node* cmp_limit, Node* bol) {
Node* new_predicate_proj = create_new_if_for_predicate(limit_check_proj, nullptr,
void PhaseIdealLoop::insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj,
Node* cmp_limit, Node* bol) {
Node* new_predicate_proj = create_new_if_for_predicate(loop_limit_check_parse_proj, nullptr,
Deoptimization::Reason_loop_limit_check,
Op_If);
Node* iff = new_predicate_proj->in(0);
@ -556,7 +557,9 @@ Node* PhaseIdealLoop::loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv,
return iv_replacement;
}
void PhaseIdealLoop::add_empty_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt) {
// Add a Parse Predicate with an uncommon trap on the failing/false path. Normal control will continue on the true path.
void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop,
SafePointNode* sfpt) {
if (!C->too_many_traps(reason)) {
Node *cont = _igvn.intcon(1);
Node* opq = new Opaque1Node(C, cont);
@ -570,7 +573,7 @@ void PhaseIdealLoop::add_empty_predicate(Deoptimization::DeoptReason reason, Nod
register_control(iffalse, _ltree_root, iff);
Node* iftrue = new IfTrueNode(iff);
register_control(iftrue, loop, iff);
C->add_predicate_opaq(opq);
C->add_parse_predicate_opaq(opq);
int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile);
address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point();
@ -864,7 +867,7 @@ bool PhaseIdealLoop::create_loop_nest(IdealLoopTree* loop, Node_List &old_new) {
julong orig_iters = (julong)hi->hi_as_long() - lo->lo_as_long();
iters_limit = checked_cast<int>(MIN2((julong)iters_limit, orig_iters));
// We need a safepoint to insert empty predicates for the inner loop.
// We need a safepoint to insert Parse Predicates for the inner loop.
SafePointNode* safepoint;
if (bt == T_INT && head->as_CountedLoop()->is_strip_mined()) {
// Loop is strip mined: use the safepoint of the outer strip mined loop
@ -1044,7 +1047,7 @@ bool PhaseIdealLoop::create_loop_nest(IdealLoopTree* loop, Node_List &old_new) {
transform_long_range_checks(checked_cast<int>(stride_con), range_checks, outer_phi, inner_iters_actual_int,
inner_phi, iv_add, inner_head);
// Peel one iteration of the loop and use the safepoint at the end
// of the peeled iteration to insert empty predicates. If no well
// of the peeled iteration to insert Parse Predicates. If no well
// positioned safepoint peel to guarantee a safepoint in the outer
// loop.
if (safepoint != nullptr || !loop->_has_call) {
@ -1058,12 +1061,12 @@ bool PhaseIdealLoop::create_loop_nest(IdealLoopTree* loop, Node_List &old_new) {
SafePointNode* cloned_sfpt = old_new[safepoint->_idx]->as_SafePoint();
if (UseLoopPredicate) {
add_empty_predicate(Deoptimization::Reason_predicate, inner_head, outer_ilt, cloned_sfpt);
add_parse_predicate(Deoptimization::Reason_predicate, inner_head, outer_ilt, cloned_sfpt);
}
if (UseProfiledLoopPredicate) {
add_empty_predicate(Deoptimization::Reason_profile_predicate, inner_head, outer_ilt, cloned_sfpt);
add_parse_predicate(Deoptimization::Reason_profile_predicate, inner_head, outer_ilt, cloned_sfpt);
}
add_empty_predicate(Deoptimization::Reason_loop_limit_check, inner_head, outer_ilt, cloned_sfpt);
add_parse_predicate(Deoptimization::Reason_loop_limit_check, inner_head, outer_ilt, cloned_sfpt);
}
#ifndef PRODUCT
@ -1785,8 +1788,9 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
assert(!x->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed");
// Generate loop's limit check.
// Loop limit check predicate should be near the loop.
ProjNode *limit_check_proj = find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check);
if (!limit_check_proj) {
ParsePredicateSuccessProj* loop_limit_check_predicate_parse_proj =
find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check);
if (loop_limit_check_predicate_parse_proj == nullptr) {
// The limit check predicate is not generated if this method trapped here before.
#ifdef ASSERT
if (TraceLoopLimitCheck) {
@ -1798,7 +1802,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
return false;
}
IfNode* check_iff = limit_check_proj->in(0)->as_If();
IfNode* check_iff = loop_limit_check_predicate_parse_proj->in(0)->as_If();
if (!is_dominator(get_ctrl(limit), check_iff->in(0))) {
return false;
@ -1815,7 +1819,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
bol = new BoolNode(cmp_limit, BoolTest::ge);
}
insert_loop_limit_check(limit_check_proj, cmp_limit, bol);
insert_loop_limit_check_predicate(loop_limit_check_predicate_parse_proj, cmp_limit, bol);
}
// Now we need to canonicalize loop condition.
@ -1828,8 +1832,9 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
// 'ne' can be replaced with 'gt' only when init > limit.
bt = BoolTest::gt;
} else {
ProjNode *limit_check_proj = find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check);
if (!limit_check_proj) {
ParsePredicateSuccessProj* loop_limit_check_predicate_parse_proj =
find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check);
if (loop_limit_check_predicate_parse_proj == nullptr) {
// The limit check predicate is not generated if this method trapped here before.
#ifdef ASSERT
if (TraceLoopLimitCheck) {
@ -1840,7 +1845,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
#endif
return false;
}
IfNode* check_iff = limit_check_proj->in(0)->as_If();
IfNode* check_iff = loop_limit_check_predicate_parse_proj->in(0)->as_If();
if (!is_dominator(get_ctrl(limit), check_iff->in(0)) ||
!is_dominator(get_ctrl(init_trip), check_iff->in(0))) {
@ -1858,7 +1863,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
bol = new BoolNode(cmp_limit, BoolTest::gt);
}
insert_loop_limit_check(limit_check_proj, cmp_limit, bol);
insert_loop_limit_check_predicate(loop_limit_check_predicate_parse_proj, cmp_limit, bol);
if (stride_con > 0) {
// 'ne' can be replaced with 'lt' only when init < limit.
@ -3953,13 +3958,13 @@ void IdealLoopTree::dump_head() {
Node* predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != nullptr ) {
tty->print(" limit_check");
entry = PhaseIdealLoop::skip_loop_predicates(entry);
entry = PhaseIdealLoop::skip_related_predicates(entry);
}
if (UseProfiledLoopPredicate) {
predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (predicate != nullptr) {
tty->print(" profile_predicated");
entry = PhaseIdealLoop::skip_loop_predicates(entry);
entry = PhaseIdealLoop::skip_related_predicates(entry);
}
}
if (UseLoopPredicate) {
@ -4078,14 +4083,14 @@ void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop,
if (predicate != nullptr) { // right pattern that can be used by loop predication
assert(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be");
useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
entry = skip_loop_predicates(entry);
entry = skip_related_predicates(entry);
}
if (UseProfiledLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (predicate != nullptr) { // right pattern that can be used by loop predication
useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
get_skeleton_predicates(entry, useful_predicates, true);
entry = skip_loop_predicates(entry);
get_assertion_predicates(entry, useful_predicates, true);
entry = skip_related_predicates(entry);
}
}
@ -4093,7 +4098,7 @@ void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop,
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != nullptr) { // right pattern that can be used by loop predication
useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
get_skeleton_predicates(entry, useful_predicates, true);
get_assertion_predicates(entry, useful_predicates, true);
}
}
}
@ -4108,7 +4113,7 @@ void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop,
// Note: it will also eliminates loop limits check predicate since it also uses
// Opaque1 node (see Parse::add_predicate()).
void PhaseIdealLoop::eliminate_useless_predicates() {
if (C->predicate_count() == 0 && C->skeleton_predicate_count() == 0) {
if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) {
return; // no predicate left
}
@ -4117,16 +4122,16 @@ void PhaseIdealLoop::eliminate_useless_predicates() {
collect_potentially_useful_predicates(_ltree_root->_child, useful_predicates);
}
for (int i = C->predicate_count(); i > 0; i--) {
Node* n = C->predicate_opaque1_node(i - 1);
for (int i = C->parse_predicate_count(); i > 0; i--) {
Node* n = C->parse_predicate_opaque1_node(i - 1);
assert(n->Opcode() == Op_Opaque1, "must be");
if (!useful_predicates.member(n)) { // not in the useful list
_igvn.replace_node(n, n->in(1));
}
}
for (int i = C->skeleton_predicate_count(); i > 0; i--) {
Node* n = C->skeleton_predicate_opaque4_node(i - 1);
for (int i = C->template_assertion_predicate_count(); i > 0; i--) {
Node* n = C->template_assertion_predicate_opaq_node(i - 1);
assert(n->Opcode() == Op_Opaque4, "must be");
if (!useful_predicates.member(n)) { // not in the useful list
_igvn.replace_node(n, n->in(2));
@ -4526,7 +4531,7 @@ void PhaseIdealLoop::build_and_optimize() {
}
// Perform loop predication before iteration splitting
if (C->has_loops() && !C->major_progress() && (C->predicate_count() > 0)) {
if (C->has_loops() && !C->major_progress() && (C->parse_predicate_count() > 0)) {
_ltree_root->_child->loop_predication(this);
}
@ -4578,8 +4583,8 @@ void PhaseIdealLoop::build_and_optimize() {
// Keep loop predicates and perform optimizations with them
// until no more loop optimizations could be done.
// After that switch predicates off and do more loop optimizations.
if (!C->major_progress() && (C->predicate_count() > 0)) {
C->cleanup_loop_predicates(_igvn);
if (!C->major_progress() && (C->parse_predicate_count() > 0)) {
C->cleanup_parse_predicates(_igvn);
if (TraceLoopOpts) {
tty->print_cr("PredicatesOff");
}

146
src/hotspot/share/opto/loopnode.hpp
View File

@ -940,25 +940,33 @@ private:
#ifdef ASSERT
void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop);
#endif
void copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new);
void copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head,
uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new);
Node* clone_skeleton_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj, Node* control,
IdealLoopTree* outer_loop, Node* input_proj);
Node* clone_skeleton_predicate_bool(Node* iff, Node* new_init, Node* new_stride, Node* control);
static bool skeleton_predicate_has_opaque(IfNode* iff);
void copy_assertion_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride, IdealLoopTree* outer_loop,
LoopNode* outer_main_head, uint dd_main_head,
uint idx_before_pre_post, uint idx_after_post_before_pre,
Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post,
const Node_List &old_new);
void copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop,
LoopNode* outer_main_head, uint dd_main_head, uint idx_before_pre_post,
uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main,
Node* zero_trip_guard_proj_post, const Node_List& old_new);
Node* clone_assertion_predicate_and_initialize(Node* iff, Node* new_init, Node* new_stride, Node* predicate,
Node* uncommon_proj, Node* control, IdealLoopTree* outer_loop,
Node* input_proj);
static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);
static bool subgraph_has_opaque(Node* n);
static void get_skeleton_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false);
void update_main_loop_skeleton_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
void copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* init, Node* stride);
void initialize_skeleton_predicates_for_peeled_loop(ProjNode* predicate, LoopNode* outer_loop_head, int dd_outer_loop_head,
Node* init, Node* stride, IdealLoopTree* outer_loop,
const uint idx_before_clone, const Node_List& old_new);
void insert_loop_limit_check(ProjNode* limit_check_proj, Node* cmp_limit, Node* bol);
Node* create_bool_from_template_assertion_predicate(Node* template_assertion_predicate, Node* new_init, Node* new_stride,
Node* control);
static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);
static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false);
void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* init,
Node* stride);
void initialize_assertion_predicates_for_peeled_loop(IfProjNode* predicate_proj, LoopNode* outer_loop_head,
const int dd_outer_loop_head, Node* init, Node* stride,
IdealLoopTree* outer_loop, const uint idx_before_clone,
const Node_List& old_new);
void insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit,
Node* bol);
#ifdef ASSERT
bool only_has_infinite_loops();
#endif
@ -1217,7 +1225,7 @@ public:
#ifdef ASSERT
bool convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* loop);
#endif
void add_empty_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt);
void add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt);
SafePointNode* find_safepoint(Node* back_control, Node* x, IdealLoopTree* loop);
IdealLoopTree* insert_outer_loop(IdealLoopTree* loop, LoopNode* outer_l, Node* outer_ift);
IdealLoopTree* create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control,
@ -1335,9 +1343,9 @@ public:
bool* p_short_scale, int depth);
// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason,
int opcode, bool rewire_uncommon_proj_phi_inputs = false,
bool if_cont_is_true_proj = true);
IfProjNode* create_new_if_for_predicate(IfProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason,
int opcode, bool rewire_uncommon_proj_phi_inputs = false,
bool if_cont_is_true_proj = true);
private:
// Helper functions for create_new_if_for_predicate()
@ -1352,47 +1360,46 @@ public:
public:
void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true);
static Node* skip_all_loop_predicates(Node* entry);
static Node* skip_loop_predicates(Node* entry);
static ProjNode* next_predicate(ProjNode* predicate);
static Node* skip_all_predicates(Node* entry);
static Node* skip_related_predicates(Node* entry);
static IfProjNode* next_predicate(IfProjNode* predicate_proj);
// Find a good location to insert a predicate
static ProjNode* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason);
static ParsePredicateSuccessProj* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason);
class Predicates {
public:
class ParsePredicates {
private:
ParsePredicateSuccessProj* _loop_predicate = nullptr;
ParsePredicateSuccessProj* _profiled_loop_predicate = nullptr;
ParsePredicateSuccessProj* _loop_limit_check_predicate = nullptr;
Node* _first_predicate = nullptr;
public:
// given loop entry, find all predicates above loop
Predicates(Node* entry);
ParsePredicates(Node* entry);
// Proj of empty loop limit check predicate
ProjNode* loop_limit_check() {
return _loop_limit_check;
// Proj of Loop Limit Check Parse Predicate.
ParsePredicateSuccessProj* loop_limit_check_predicate() {
return _loop_limit_check_predicate;
}
// Proj of empty profile predicate
ProjNode* profile_predicate() {
return _profile_predicate;
// Proj of Profile Loop Parse Predicate.
ParsePredicateSuccessProj* profiled_loop_predicate() {
return _profiled_loop_predicate;
}
// Proj of empty predicate
ProjNode* predicate() {
return _predicate;
// Proj of Loop Parse Predicate.
ParsePredicateSuccessProj* loop_predicate() {
return _loop_predicate;
}
// First control node above all predicates
Node* skip_all() {
return _entry_to_all_predicates;
// Proj of first Parse Predicate when walking the graph down from root.
Node* get_first_predicate() {
return _first_predicate;
}
private:
ProjNode*_loop_limit_check = nullptr;
ProjNode* _profile_predicate = nullptr;
ProjNode* _predicate = nullptr;
Node* _entry_to_all_predicates = nullptr;
};
// Find a predicate
static Node* find_predicate(Node* entry);
static Node* find_parse_predicate(Node* entry);
// Construct a range check for a predicate if
BoolNode* rc_predicate(IdealLoopTree *loop, Node* ctrl,
int scale, Node* offset,
@ -1402,23 +1409,18 @@ public:
// Implementation of the loop predication to promote checks outside the loop
bool loop_predication_impl(IdealLoopTree *loop);
bool loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* proj, ProjNode *predicate_proj,
CountedLoopNode *cl, ConNode* zero, Invariance& invar,
Deoptimization::DeoptReason reason);
bool loop_predication_should_follow_branches(IdealLoopTree *loop, ProjNode *predicate_proj, float& loop_trip_cnt);
bool loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_proj,
ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, ConNode* zero,
Invariance& invar, Deoptimization::DeoptReason reason);
bool loop_predication_should_follow_branches(IdealLoopTree* loop, IfProjNode* predicate_proj, float& loop_trip_cnt);
void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt,
PathFrequency& pf, Node_Stack& stack, VectorSet& seen,
Node_List& if_proj_list);
ProjNode* insert_initial_skeleton_predicate(IfNode* iff, IdealLoopTree *loop,
ProjNode* proj, ProjNode *predicate_proj,
ProjNode* upper_bound_proj,
int scale, Node* offset,
Node* init, Node* limit, jint stride,
Node* rng, bool& overflow,
Deoptimization::DeoptReason reason);
Node* add_range_check_predicate(IdealLoopTree* loop, CountedLoopNode* cl,
Node* predicate_proj, int scale_con, Node* offset,
Node* limit, jint stride_con, Node* value);
IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, IfProjNode* predicate_proj,
IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit,
jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason);
Node* add_range_check_elimination_assertion_predicate(IdealLoopTree* loop, Node* predicate_proj, int scale_con,
Node* offset, Node* limit, jint stride_con, Node* value);
// Helper function to collect predicate for eliminating the useless ones
void collect_potentially_useful_predicates(IdealLoopTree *loop, Unique_Node_List &predicate_opaque1);
@ -1664,16 +1666,18 @@ private:
_nodes_required = UINT_MAX;
}
// Clone loop predicates to slow and fast loop when unswitching a loop
void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred);
ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, bool slow_loop);
void clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason,
ProjNode* old_predicate_proj, ProjNode* iffast_pred, ProjNode* ifslow_pred);
ProjNode* clone_skeleton_predicate_for_unswitched_loops(Node* iff, ProjNode* predicate,
Deoptimization::DeoptReason reason,
ProjNode* output_proj);
static void check_created_predicate_for_unswitching(const Node* new_entry) PRODUCT_RETURN;
// Clone Parse Predicates to slow and fast loop when unswitching a loop
void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, bool slow_loop);
void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
Deoptimization::DeoptReason reason, IfProjNode* old_predicate_proj,
IfProjNode* iffast_pred, IfProjNode* ifslow_pred);
IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
Deoptimization::DeoptReason reason,
IfProjNode* output_proj);
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry) PRODUCT_RETURN;
bool _created_loop_node;
DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);)

8
src/hotspot/share/opto/loopopts.cpp
View File

@ -3287,7 +3287,7 @@ bool PhaseIdealLoop::is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& p
// stmt1
// |
// v
// loop predicate
// predicates
// |
// v
// loop<----+
@ -3322,7 +3322,7 @@ bool PhaseIdealLoop::is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& p
// stmt1
// |
// v
// loop predicate
// predicates
// / \
// clone / \ orig
// / \
@ -3363,7 +3363,7 @@ bool PhaseIdealLoop::is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& p
// stmt1
// |
// v
// loop predicate
// predicates
// /
// clone / orig
// / TOP
@ -3407,7 +3407,7 @@ bool PhaseIdealLoop::is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& p
// stmt1
// |
// v
// loop predicate
// predicates
// |
// v
// stmt2 clone

2
src/hotspot/share/opto/node.cpp
View File

@ -607,7 +607,7 @@ void Node::destruct(PhaseValues* phase) {
compile->remove_expensive_node(this);
}
if (Opcode() == Op_Opaque4) {
compile->remove_skeleton_predicate_opaq(this);
compile->remove_template_assertion_predicate_opaq(this);
}
if (for_post_loop_opts_igvn()) {
compile->remove_from_post_loop_opts_igvn(this);

6
src/hotspot/share/opto/parse1.cpp
View File

@ -675,7 +675,7 @@ void Parse::do_all_blocks() {
// Need correct bci for predicate.
// It is fine to set it here since do_one_block() will set it anyway.
set_parse_bci(block->start());
add_empty_predicates();
add_parse_predicates();
}
// Add new region for back branches.
int edges = block->pred_count() - block->preds_parsed() + 1; // +1 for original region
@ -1695,9 +1695,9 @@ void Parse::merge_common(Parse::Block* target, int pnum) {
if (target->is_SEL_head()) {
DEBUG_ONLY( target->mark_merged_backedge(block()); )
if (target->start() == 0) {
// Add loop predicate for the special case when
// Add Parse Predicates for the special case when
// there are backbranches to the method entry.
add_empty_predicates();
add_parse_predicates();
}
}
// Add a Region to start the new basic block. Phis will be added

2
src/hotspot/share/opto/parse2.cpp
View File

@ -1580,7 +1580,7 @@ void Parse::maybe_add_predicate_after_if(Block* path) {
// Add predicates at bci of if dominating the loop so traps can be
// recorded on the if's profile data
int bc_depth = repush_if_args();
add_empty_predicates();
add_parse_predicates();
dec_sp(bc_depth);
path->set_has_predicates();
}

4
src/hotspot/share/opto/split_if.cpp
View File

@ -101,8 +101,8 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) {
for (uint i = 0; i < wq.size(); i++) {
Node* m = wq.at(i);
if (m->is_If()) {
assert(skeleton_predicate_has_opaque(m->as_If()), "opaque node not reachable from if?");
Node* bol = clone_skeleton_predicate_bool(m, nullptr, nullptr, m->in(0));
assert(assertion_predicate_has_loop_opaque_node(m->as_If()), "opaque node not reachable from if?");
Node* bol = create_bool_from_template_assertion_predicate(m, nullptr, nullptr, m->in(0));
_igvn.replace_input_of(m, 1, bol);
} else {
assert(!m->is_CFG(), "not CFG expected");

6
src/hotspot/share/opto/stringopts.cpp
View File

@ -1214,7 +1214,7 @@ Node* PhaseStringOpts::int_stringSize(GraphKit& kit, Node* arg) {
Node* final_size = new PhiNode(final_merge, TypeInt::INT);
kit.gvn().set_type(final_size, TypeInt::INT);
kit.add_empty_predicates();
kit.add_parse_predicates();
C->set_has_loops(true);
RegionNode* loop = new RegionNode(3);
@ -1299,8 +1299,8 @@ void PhaseStringOpts::getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicT
// if (i == 0) break;
// }
// Add loop predicate first.
kit.add_empty_predicates();
// Add Parse Predicates first.
kit.add_parse_predicates();
C->set_has_loops(true);
RegionNode* head = new RegionNode(3);