8305637: Remove Opaque1 nodes for Parse Predicates and clean up useless predicate elimination
Reviewed-by: roland, kvn
This commit is contained in:
parent
8647f001bb
commit
4b445753e6
@ -734,7 +734,7 @@ Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one
|
||||
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
|
||||
Node* u = n->fast_out(j);
|
||||
if (u->is_CFG()) {
|
||||
if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
|
||||
if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj()) {
|
||||
return NodeSentinel;
|
||||
}
|
||||
}
|
||||
@ -743,7 +743,7 @@ Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one
|
||||
}
|
||||
} else if (c->is_Proj()) {
|
||||
if (c->is_IfProj()) {
|
||||
if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != nullptr) {
|
||||
if (c->as_Proj()->is_uncommon_trap_if_pattern() != nullptr) {
|
||||
// continue;
|
||||
} else {
|
||||
if (!allow_one_proj) {
|
||||
@ -1138,7 +1138,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
|
||||
|
||||
if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
|
||||
u->in(0)->Opcode() == Op_IfTrue &&
|
||||
u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
|
||||
u->in(0)->as_Proj()->is_uncommon_trap_if_pattern() &&
|
||||
u->in(0)->in(0)->is_If() &&
|
||||
u->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
|
||||
u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&
|
||||
|
@ -463,8 +463,9 @@ public:
|
||||
// More information about predicates can be found in loopPredicate.cpp.
|
||||
class ParsePredicateNode : public IfNode {
|
||||
Deoptimization::DeoptReason _deopt_reason;
|
||||
bool _useless; // If the associated loop dies, this parse predicate becomes useless and can be cleaned up by Value().
|
||||
public:
|
||||
ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason);
|
||||
ParsePredicateNode(Node* control, Deoptimization::DeoptReason deopt_reason, PhaseGVN* gvn);
|
||||
virtual int Opcode() const;
|
||||
virtual uint size_of() const { return sizeof(*this); }
|
||||
|
||||
@ -472,8 +473,25 @@ class ParsePredicateNode : public IfNode {
|
||||
return _deopt_reason;
|
||||
}
|
||||
|
||||
bool is_useless() const {
|
||||
return _useless;
|
||||
}
|
||||
|
||||
void mark_useless() {
|
||||
_useless = true;
|
||||
}
|
||||
|
||||
void mark_useful() {
|
||||
_useless = false;
|
||||
}
|
||||
|
||||
Node* uncommon_trap() const;
|
||||
|
||||
Node* Ideal(PhaseGVN* phase, bool can_reshape) {
|
||||
return nullptr; // Don't optimize
|
||||
}
|
||||
|
||||
const Type* Value(PhaseGVN* phase) const;
|
||||
NOT_PRODUCT(void dump_spec(outputStream* st) const;)
|
||||
};
|
||||
|
||||
|
@ -360,10 +360,11 @@ void Compile::remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines
|
||||
assert(found <= 1, "not unique");
|
||||
}
|
||||
|
||||
void Compile::remove_useless_nodes(GrowableArray<Node*>& node_list, Unique_Node_List& useful) {
|
||||
template<typename N, ENABLE_IF_SDEFN(std::is_base_of<Node, N>::value)>
|
||||
void Compile::remove_useless_nodes(GrowableArray<N*>& node_list, Unique_Node_List& useful) {
|
||||
for (int i = node_list.length() - 1; i >= 0; i--) {
|
||||
Node* n = node_list.at(i);
|
||||
if (!useful.member(n)) {
|
||||
N* node = node_list.at(i);
|
||||
if (!useful.member(node)) {
|
||||
node_list.delete_at(i); // replaces i-th with last element which is known to be useful (already processed)
|
||||
}
|
||||
}
|
||||
@ -389,6 +390,9 @@ void Compile::remove_useless_node(Node* dead) {
|
||||
if (dead->Opcode() == Op_Opaque4) {
|
||||
remove_template_assertion_predicate_opaq(dead);
|
||||
}
|
||||
if (dead->is_ParsePredicate()) {
|
||||
remove_parse_predicate(dead->as_ParsePredicate());
|
||||
}
|
||||
if (dead->for_post_loop_opts_igvn()) {
|
||||
remove_from_post_loop_opts_igvn(dead);
|
||||
}
|
||||
@ -436,7 +440,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis
|
||||
}
|
||||
|
||||
remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
|
||||
remove_useless_nodes(_parse_predicate_opaqs, useful); // remove useless Parse Predicate opaque nodes
|
||||
remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes
|
||||
remove_useless_nodes(_template_assertion_predicate_opaqs, useful); // remove useless Assertion Predicate opaque nodes
|
||||
remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
|
||||
remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
|
||||
@ -631,7 +635,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci,
|
||||
_failure_reason(nullptr),
|
||||
_intrinsics (comp_arena(), 0, 0, nullptr),
|
||||
_macro_nodes (comp_arena(), 8, 0, nullptr),
|
||||
_parse_predicate_opaqs (comp_arena(), 8, 0, nullptr),
|
||||
_parse_predicates (comp_arena(), 8, 0, nullptr),
|
||||
_template_assertion_predicate_opaqs (comp_arena(), 8, 0, nullptr),
|
||||
_expensive_nodes (comp_arena(), 8, 0, nullptr),
|
||||
_for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
|
||||
@ -1807,18 +1811,18 @@ bool Compile::can_alias(const TypePtr* adr_type, int alias_idx) {
|
||||
return adr_idx == alias_idx;
|
||||
}
|
||||
|
||||
// Remove the opaque nodes that protect the Parse Predicates so that all unused
|
||||
// checks and uncommon_traps will be eliminated from the ideal graph.
|
||||
void Compile::cleanup_parse_predicates(PhaseIterGVN& igvn) const {
|
||||
// Mark all ParsePredicateNodes as useless. They will later be removed from the graph in IGVN together with their
|
||||
// uncommon traps if no Runtime Predicates were created from the Parse Predicates.
|
||||
void Compile::mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn) {
|
||||
if (parse_predicate_count() == 0) {
|
||||
return;
|
||||
}
|
||||
for (int i = parse_predicate_count(); i > 0; i--) {
|
||||
Node* n = parse_predicate_opaque1_node(i - 1);
|
||||
assert(n->Opcode() == Op_Opaque1, "must be");
|
||||
igvn.replace_node(n, n->in(1));
|
||||
for (int i = 0; i < parse_predicate_count(); i++) {
|
||||
ParsePredicateNode* parse_predicate = _parse_predicates.at(i);
|
||||
parse_predicate->mark_useless();
|
||||
igvn._worklist.push(parse_predicate);
|
||||
}
|
||||
assert(parse_predicate_count() == 0, "should be clean!");
|
||||
_parse_predicates.clear();
|
||||
}
|
||||
|
||||
void Compile::record_for_post_loop_opts_igvn(Node* n) {
|
||||
@ -1851,6 +1855,7 @@ void Compile::process_for_post_loop_opts_igvn(PhaseIterGVN& igvn) {
|
||||
}
|
||||
igvn.optimize();
|
||||
assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
|
||||
assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
|
||||
|
||||
// Sometimes IGVN sets major progress (e.g., when processing loop nodes).
|
||||
if (C->major_progress()) {
|
||||
|
@ -69,6 +69,7 @@ class Node_Notes;
|
||||
class NodeHash;
|
||||
class NodeCloneInfo;
|
||||
class OptoReg;
|
||||
class ParsePredicateNode;
|
||||
class PhaseCFG;
|
||||
class PhaseGVN;
|
||||
class PhaseIterGVN;
|
||||
@ -357,7 +358,7 @@ class Compile : public Phase {
|
||||
const char* _failure_reason; // for record_failure/failing pattern
|
||||
GrowableArray<CallGenerator*> _intrinsics; // List of intrinsics.
|
||||
GrowableArray<Node*> _macro_nodes; // List of nodes which need to be expanded before matching.
|
||||
GrowableArray<Node*> _parse_predicate_opaqs; // List of Opaque1 nodes for the Parse Predicates.
|
||||
GrowableArray<ParsePredicateNode*> _parse_predicates; // List of Parse Predicates.
|
||||
GrowableArray<Node*> _template_assertion_predicate_opaqs; // List of Opaque4 nodes for Template Assertion Predicates.
|
||||
GrowableArray<Node*> _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common
|
||||
GrowableArray<Node*> _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over
|
||||
@ -703,13 +704,13 @@ private:
|
||||
#endif
|
||||
|
||||
int macro_count() const { return _macro_nodes.length(); }
|
||||
int parse_predicate_count() const { return _parse_predicate_opaqs.length(); }
|
||||
int parse_predicate_count() const { return _parse_predicates.length(); }
|
||||
int template_assertion_predicate_count() const { return _template_assertion_predicate_opaqs.length(); }
|
||||
int expensive_count() const { return _expensive_nodes.length(); }
|
||||
int coarsened_count() const { return _coarsened_locks.length(); }
|
||||
|
||||
Node* macro_node(int idx) const { return _macro_nodes.at(idx); }
|
||||
Node* parse_predicate_opaque1_node(int idx) const { return _parse_predicate_opaqs.at(idx); }
|
||||
ParsePredicateNode* parse_predicate(int idx) const { return _parse_predicates.at(idx); }
|
||||
|
||||
Node* template_assertion_predicate_opaq_node(int idx) const {
|
||||
return _template_assertion_predicate_opaqs.at(idx);
|
||||
@ -728,10 +729,6 @@ private:
|
||||
// this function may be called twice for a node so we can only remove it
|
||||
// if it's still existing.
|
||||
_macro_nodes.remove_if_existing(n);
|
||||
// remove from _parse_predicate_opaqs list also if it is there
|
||||
if (parse_predicate_count() > 0) {
|
||||
_parse_predicate_opaqs.remove_if_existing(n);
|
||||
}
|
||||
// Remove from coarsened locks list if present
|
||||
if (coarsened_count() > 0) {
|
||||
remove_coarsened_lock(n);
|
||||
@ -741,16 +738,24 @@ private:
|
||||
void remove_expensive_node(Node* n) {
|
||||
_expensive_nodes.remove_if_existing(n);
|
||||
}
|
||||
void add_parse_predicate_opaq(Node* n) {
|
||||
assert(!_parse_predicate_opaqs.contains(n), "duplicate entry in Parse Predicate opaque1 list");
|
||||
assert(_macro_nodes.contains(n), "should have already been in macro list");
|
||||
_parse_predicate_opaqs.append(n);
|
||||
|
||||
void add_parse_predicate(ParsePredicateNode* n) {
|
||||
assert(!_parse_predicates.contains(n), "duplicate entry in Parse Predicate list");
|
||||
_parse_predicates.append(n);
|
||||
}
|
||||
|
||||
void remove_parse_predicate(ParsePredicateNode* n) {
|
||||
if (parse_predicate_count() > 0) {
|
||||
_parse_predicates.remove_if_existing(n);
|
||||
}
|
||||
}
|
||||
|
||||
void add_template_assertion_predicate_opaq(Node* n) {
|
||||
assert(!_template_assertion_predicate_opaqs.contains(n),
|
||||
"duplicate entry in template assertion predicate opaque4 list");
|
||||
_template_assertion_predicate_opaqs.append(n);
|
||||
}
|
||||
|
||||
void remove_template_assertion_predicate_opaq(Node* n) {
|
||||
if (template_assertion_predicate_count() > 0) {
|
||||
_template_assertion_predicate_opaqs.remove_if_existing(n);
|
||||
@ -775,12 +780,7 @@ private:
|
||||
|
||||
void sort_macro_nodes();
|
||||
|
||||
// Remove the opaque nodes that protect the Parse Predicates so that the unused checks and
|
||||
// uncommon traps will be eliminated from the graph.
|
||||
void cleanup_parse_predicates(PhaseIterGVN &igvn) const;
|
||||
bool is_predicate_opaq(Node* n) const {
|
||||
return _parse_predicate_opaqs.contains(n);
|
||||
}
|
||||
void mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn);
|
||||
|
||||
// Are there candidate expensive nodes for optimization?
|
||||
bool should_optimize_expensive_nodes(PhaseIterGVN &igvn);
|
||||
@ -1021,7 +1021,8 @@ private:
|
||||
_vector_reboxing_late_inlines.push(cg);
|
||||
}
|
||||
|
||||
void remove_useless_nodes (GrowableArray<Node*>& node_list, Unique_Node_List &useful);
|
||||
template<typename N, ENABLE_IF(std::is_base_of<Node, N>::value)>
|
||||
void remove_useless_nodes(GrowableArray<N*>& node_list, Unique_Node_List& useful);
|
||||
|
||||
void remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Unique_Node_List &useful);
|
||||
void remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Node* dead);
|
||||
|
@ -3996,11 +3996,7 @@ void GraphKit::add_parse_predicate(Deoptimization::DeoptReason reason, const int
|
||||
return;
|
||||
}
|
||||
|
||||
Node* cont = _gvn.intcon(1);
|
||||
Node* opaq = _gvn.transform(new Opaque1Node(C, cont));
|
||||
C->add_parse_predicate_opaq(opaq);
|
||||
Node* bol = _gvn.transform(new Conv2BNode(opaq));
|
||||
ParsePredicateNode* parse_predicate = new ParsePredicateNode(control(), bol, reason);
|
||||
ParsePredicateNode* parse_predicate = new ParsePredicateNode(control(), reason, &_gvn);
|
||||
_gvn.set_type(parse_predicate, parse_predicate->Value(&_gvn));
|
||||
Node* if_false = _gvn.transform(new IfFalseNode(parse_predicate));
|
||||
{
|
||||
|
@ -800,7 +800,7 @@ bool IfNode::is_dominator_unc(CallStaticJavaNode* dom_unc, CallStaticJavaNode* u
|
||||
// Return projection that leads to an uncommon trap if any
|
||||
ProjNode* IfNode::uncommon_trap_proj(CallStaticJavaNode*& call) const {
|
||||
for (int i = 0; i < 2; i++) {
|
||||
call = proj_out(i)->is_uncommon_trap_proj(Deoptimization::Reason_none);
|
||||
call = proj_out(i)->is_uncommon_trap_proj();
|
||||
if (call != nullptr) {
|
||||
return proj_out(i);
|
||||
}
|
||||
@ -811,7 +811,7 @@ ProjNode* IfNode::uncommon_trap_proj(CallStaticJavaNode*& call) const {
|
||||
// Do this If and the dominating If both branch out to an uncommon trap
|
||||
bool IfNode::has_only_uncommon_traps(ProjNode* proj, ProjNode*& success, ProjNode*& fail, PhaseIterGVN* igvn) {
|
||||
ProjNode* otherproj = proj->other_if_proj();
|
||||
CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj();
|
||||
|
||||
if (otherproj->outcnt() == 1 && dom_unc != nullptr) {
|
||||
// We need to re-execute the folded Ifs after deoptimization from the merged traps
|
||||
@ -1076,8 +1076,8 @@ Node* IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode*
|
||||
|
||||
ProjNode* otherproj = proj->other_if_proj();
|
||||
|
||||
CallStaticJavaNode* unc = success->is_uncommon_trap_proj(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* unc = success->is_uncommon_trap_proj();
|
||||
CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj();
|
||||
|
||||
if (unc != dom_unc) {
|
||||
Node* r = new RegionNode(3);
|
||||
@ -1241,13 +1241,13 @@ bool IfNode::is_side_effect_free_test(ProjNode* proj, PhaseIterGVN* igvn) {
|
||||
if (proj == nullptr) {
|
||||
return false;
|
||||
}
|
||||
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern();
|
||||
if (unc != nullptr && proj->outcnt() <= 2) {
|
||||
if (proj->outcnt() == 1 ||
|
||||
// Allow simple null check from LoadRange
|
||||
(is_cmp_with_loadrange(proj) && is_null_check(proj, igvn))) {
|
||||
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* dom_unc = proj->in(0)->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern();
|
||||
CallStaticJavaNode* dom_unc = proj->in(0)->in(0)->as_Proj()->is_uncommon_trap_if_pattern();
|
||||
assert(dom_unc != nullptr, "is_uncommon_trap_if_pattern returned null");
|
||||
|
||||
// reroute_side_effect_free_unc changes the state of this
|
||||
@ -1278,9 +1278,9 @@ bool IfNode::is_side_effect_free_test(ProjNode* proj, PhaseIterGVN* igvn) {
|
||||
// where the first CmpI would have prevented it from executing: on a
|
||||
// trap, we need to restart execution at the state of the first CmpI
|
||||
void IfNode::reroute_side_effect_free_unc(ProjNode* proj, ProjNode* dom_proj, PhaseIterGVN* igvn) {
|
||||
CallStaticJavaNode* dom_unc = dom_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* dom_unc = dom_proj->is_uncommon_trap_if_pattern();
|
||||
ProjNode* otherproj = proj->other_if_proj();
|
||||
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern();
|
||||
Node* call_proj = dom_unc->unique_ctrl_out();
|
||||
Node* halt = call_proj->unique_ctrl_out();
|
||||
|
||||
@ -1975,11 +1975,13 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) {
|
||||
return dominated_by(prev_dom, igvn);
|
||||
}
|
||||
|
||||
ParsePredicateNode::ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason)
|
||||
: IfNode(control, bol, PROB_MAX, COUNT_UNKNOWN),
|
||||
_deopt_reason(deopt_reason) {
|
||||
ParsePredicateNode::ParsePredicateNode(Node* control, Deoptimization::DeoptReason deopt_reason, PhaseGVN* gvn)
|
||||
: IfNode(control, gvn->intcon(1), PROB_MAX, COUNT_UNKNOWN),
|
||||
_deopt_reason(deopt_reason),
|
||||
_useless(false) {
|
||||
init_class_id(Class_ParsePredicate);
|
||||
assert(bol->Opcode() == Op_Conv2B && bol->in(1) != nullptr && bol->in(1)->is_Opaque1(), "wrong boolean input");
|
||||
gvn->C->add_parse_predicate(this);
|
||||
gvn->C->record_for_post_loop_opts_igvn(this);
|
||||
#ifdef ASSERT
|
||||
switch (deopt_reason) {
|
||||
case Deoptimization::Reason_predicate:
|
||||
@ -1999,6 +2001,18 @@ Node* ParsePredicateNode::uncommon_trap() const {
|
||||
return uct_region_or_call;
|
||||
}
|
||||
|
||||
// Fold this node away once it becomes useless or at latest in post loop opts IGVN.
|
||||
const Type* ParsePredicateNode::Value(PhaseGVN* phase) const {
|
||||
if (phase->type(in(0)) == Type::TOP) {
|
||||
return Type::TOP;
|
||||
}
|
||||
if (_useless || phase->C->post_loop_opts_phase()) {
|
||||
return TypeTuple::IFTRUE;
|
||||
} else {
|
||||
return bottom_type();
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef PRODUCT
|
||||
void ParsePredicateNode::dump_spec(outputStream* st) const {
|
||||
st->print(" #");
|
||||
|
@ -99,43 +99,43 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred,
|
||||
// We will create a region to guard the uct call if there is no one there.
|
||||
// The continuation projection (if_cont) of the new_iff is returned which
|
||||
// is an IfTrue projection. This code is also used to clone predicates to cloned loops.
|
||||
IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, Node* new_entry,
|
||||
IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
|
||||
Deoptimization::DeoptReason reason,
|
||||
const int opcode, const bool rewire_uncommon_proj_phi_inputs) {
|
||||
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
|
||||
IfNode* iff = cont_proj->in(0)->as_If();
|
||||
assert(parse_predicate_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
|
||||
ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate();
|
||||
|
||||
ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
|
||||
Node *rgn = uncommon_proj->unique_ctrl_out();
|
||||
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
|
||||
ProjNode* uncommon_proj = parse_predicate->proj_out(false);
|
||||
Node* uct_region = uncommon_proj->unique_ctrl_out();
|
||||
assert(uct_region->is_Region() || uct_region->is_Call(), "must be a region or call uct");
|
||||
|
||||
uint proj_index = 1; // region's edge corresponding to uncommon_proj
|
||||
if (!rgn->is_Region()) { // create a region to guard the call
|
||||
assert(rgn->is_Call(), "must be call uct");
|
||||
CallNode* call = rgn->as_Call();
|
||||
if (!uct_region->is_Region()) { // create a region to guard the call
|
||||
assert(uct_region->is_Call(), "must be call uct");
|
||||
CallNode* call = uct_region->as_Call();
|
||||
IdealLoopTree* loop = get_loop(call);
|
||||
rgn = new RegionNode(1);
|
||||
uct_region = new RegionNode(1);
|
||||
Node* uncommon_proj_orig = uncommon_proj;
|
||||
uncommon_proj = uncommon_proj->clone()->as_Proj();
|
||||
register_control(uncommon_proj, loop, iff);
|
||||
rgn->add_req(uncommon_proj);
|
||||
register_control(rgn, loop, uncommon_proj);
|
||||
_igvn.replace_input_of(call, 0, rgn);
|
||||
register_control(uncommon_proj, loop, parse_predicate);
|
||||
uct_region->add_req(uncommon_proj);
|
||||
register_control(uct_region, loop, uncommon_proj);
|
||||
_igvn.replace_input_of(call, 0, uct_region);
|
||||
// When called from beautify_loops() idom is not constructed yet.
|
||||
if (_idom != nullptr) {
|
||||
set_idom(call, rgn, dom_depth(rgn));
|
||||
set_idom(call, uct_region, dom_depth(uct_region));
|
||||
}
|
||||
// Move nodes pinned on the projection or whose control is set to
|
||||
// the projection to the region.
|
||||
lazy_replace(uncommon_proj_orig, rgn);
|
||||
lazy_replace(uncommon_proj_orig, uct_region);
|
||||
} else {
|
||||
// Find region's edge corresponding to uncommon_proj
|
||||
for (; proj_index < rgn->req(); proj_index++)
|
||||
if (rgn->in(proj_index) == uncommon_proj) break;
|
||||
assert(proj_index < rgn->req(), "sanity");
|
||||
for (; proj_index < uct_region->req(); proj_index++)
|
||||
if (uct_region->in(proj_index) == uncommon_proj) break;
|
||||
assert(proj_index < uct_region->req(), "sanity");
|
||||
}
|
||||
|
||||
Node* entry = iff->in(0);
|
||||
Node* entry = parse_predicate->in(0);
|
||||
if (new_entry != nullptr) {
|
||||
// Cloning the predicate to new location.
|
||||
entry = new_entry;
|
||||
@ -145,13 +145,13 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N
|
||||
IfNode* new_iff = nullptr;
|
||||
switch (opcode) {
|
||||
case Op_If:
|
||||
new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
|
||||
new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt);
|
||||
break;
|
||||
case Op_RangeCheck:
|
||||
new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
|
||||
new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt);
|
||||
break;
|
||||
case Op_ParsePredicate:
|
||||
new_iff = new ParsePredicateNode(entry, iff->in(1), reason);
|
||||
new_iff = new ParsePredicateNode(entry, reason, &_igvn);
|
||||
break;
|
||||
default:
|
||||
fatal("no other If variant here");
|
||||
@ -160,23 +160,19 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N
|
||||
IfProjNode* if_cont = new IfTrueNode(new_iff);
|
||||
IfProjNode* if_uct = new IfFalseNode(new_iff);
|
||||
|
||||
if (cont_proj->is_IfFalse()) {
|
||||
// Swap
|
||||
IfProjNode* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
|
||||
}
|
||||
register_control(if_cont, lp, new_iff);
|
||||
register_control(if_uct, get_loop(rgn), new_iff);
|
||||
register_control(if_uct, get_loop(uct_region), new_iff);
|
||||
|
||||
_igvn.add_input_to(rgn, if_uct);
|
||||
_igvn.add_input_to(uct_region, if_uct);
|
||||
|
||||
// If rgn has phis add new edges which has the same
|
||||
// value as on original uncommon_proj pass.
|
||||
assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last");
|
||||
assert(uct_region->in(uct_region->req() - 1) == if_uct, "new edge should be last");
|
||||
bool has_phi = false;
|
||||
for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) {
|
||||
Node* use = rgn->fast_out(i);
|
||||
for (DUIterator_Fast imax, i = uct_region->fast_outs(imax); i < imax; i++) {
|
||||
Node* use = uct_region->fast_out(i);
|
||||
if (use->is_Phi() && use->outcnt() > 0) {
|
||||
assert(use->in(0) == rgn, "");
|
||||
assert(use->in(0) == uct_region, "");
|
||||
_igvn.rehash_node_delayed(use);
|
||||
Node* phi_input = use->in(proj_index);
|
||||
|
||||
@ -197,21 +193,21 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N
|
||||
has_phi = true;
|
||||
}
|
||||
}
|
||||
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
|
||||
assert(!has_phi || uct_region->req() > 3, "no phis when region is created");
|
||||
|
||||
if (new_entry == nullptr) {
|
||||
// Attach if_cont to iff
|
||||
_igvn.replace_input_of(iff, 0, if_cont);
|
||||
_igvn.replace_input_of(parse_predicate, 0, if_cont);
|
||||
if (_idom != nullptr) {
|
||||
set_idom(iff, if_cont, dom_depth(iff));
|
||||
set_idom(parse_predicate, if_cont, dom_depth(parse_predicate));
|
||||
}
|
||||
}
|
||||
|
||||
// When called from beautify_loops() idom is not constructed yet.
|
||||
if (_idom != nullptr) {
|
||||
Node* ridom = idom(rgn);
|
||||
Node* ridom = idom(uct_region);
|
||||
Node* nrdom = dom_lca_internal(ridom, new_iff);
|
||||
set_idom(rgn, nrdom, dom_depth(rgn));
|
||||
set_idom(uct_region, nrdom, dom_depth(uct_region));
|
||||
}
|
||||
|
||||
return if_cont->as_IfProj();
|
||||
@ -303,24 +299,14 @@ void PhaseIdealLoop::rewire_inputs_of_clones_to_clones(Node* new_ctrl, Node* clo
|
||||
}
|
||||
}
|
||||
|
||||
IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* predicate_proj,
|
||||
IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj,
|
||||
Node* new_entry, Deoptimization::DeoptReason reason,
|
||||
const bool slow_loop) {
|
||||
|
||||
IfProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_ParsePredicate,
|
||||
IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate,
|
||||
slow_loop);
|
||||
IfNode* iff = new_predicate_proj->in(0)->as_If();
|
||||
Node* ctrl = iff->in(0);
|
||||
|
||||
// Match original condition since predicate's projections could be swapped.
|
||||
assert(predicate_proj->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
|
||||
Node* opq = new Opaque1Node(C, predicate_proj->in(0)->in(1)->in(1)->in(1));
|
||||
C->add_parse_predicate_opaq(opq);
|
||||
Node* bol = new Conv2BNode(opq);
|
||||
register_new_node(opq, ctrl);
|
||||
register_new_node(bol, ctrl);
|
||||
_igvn.hash_delete(iff);
|
||||
iff->set_req(1, bol);
|
||||
assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection");
|
||||
ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate();
|
||||
return new_predicate_proj;
|
||||
}
|
||||
|
||||
@ -329,9 +315,11 @@ IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredic
|
||||
// cloned predicates.
|
||||
void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
|
||||
Deoptimization::DeoptReason reason,
|
||||
IfProjNode* old_predicate_proj, IfProjNode* iffast_pred,
|
||||
IfProjNode* ifslow_pred) {
|
||||
assert(iffast_pred->in(0)->is_If() && ifslow_pred->in(0)->is_If(), "sanity check");
|
||||
IfProjNode* old_predicate_proj,
|
||||
ParsePredicateSuccessProj* fast_loop_parse_predicate_proj,
|
||||
ParsePredicateSuccessProj* slow_loop_parse_predicate_proj) {
|
||||
assert(fast_loop_parse_predicate_proj->in(0)->is_ParsePredicate() &&
|
||||
slow_loop_parse_predicate_proj->in(0)->is_ParsePredicate(), "sanity check");
|
||||
// Only need to clone range check predicates as those can be changed and duplicated by inserting pre/main/post loops
|
||||
// and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the
|
||||
// original predicate order.
|
||||
@ -350,9 +338,9 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree
|
||||
assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj(), "predicate must be a projection of an if node");
|
||||
IfProjNode* predicate_proj = predicate->as_IfProj();
|
||||
|
||||
IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, iffast_pred);
|
||||
IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, fast_loop_parse_predicate_proj);
|
||||
assert(assertion_predicate_has_loop_opaque_node(fast_proj->in(0)->as_If()), "must find Assertion Predicate for fast loop");
|
||||
IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, ifslow_pred);
|
||||
IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, slow_loop_parse_predicate_proj);
|
||||
assert(assertion_predicate_has_loop_opaque_node(slow_proj->in(0)->as_If()), "must find Assertion Predicate for slow loop");
|
||||
|
||||
// Update control dependent data nodes.
|
||||
@ -378,14 +366,13 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree
|
||||
// Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
|
||||
// is set, then the Opaque4 nodes of the Assertion Predicates are put on the list instead of the projections.
|
||||
void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) {
|
||||
IfNode* iff = predicate->in(0)->as_If();
|
||||
ProjNode* uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
|
||||
ParsePredicateNode* parse_predicate = predicate->in(0)->as_ParsePredicate();
|
||||
ProjNode* uncommon_proj = parse_predicate->proj_out(1 - predicate->as_Proj()->_con);
|
||||
Node* rgn = uncommon_proj->unique_ctrl_out();
|
||||
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
|
||||
assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape");
|
||||
predicate = iff->in(0);
|
||||
predicate = parse_predicate->in(0);
|
||||
while (predicate != nullptr && predicate->is_Proj() && predicate->in(0)->is_If()) {
|
||||
iff = predicate->in(0)->as_If();
|
||||
IfNode* iff = predicate->in(0)->as_If();
|
||||
uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
|
||||
if (uncommon_proj->unique_ctrl_out() != rgn) {
|
||||
break;
|
||||
@ -408,12 +395,12 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List&
|
||||
// predicate again).
|
||||
IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
|
||||
Deoptimization::DeoptReason reason,
|
||||
IfProjNode* output_proj) {
|
||||
Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, output_proj);
|
||||
IfProjNode* if_proj = create_new_if_for_predicate(output_proj, nullptr, reason, iff->Opcode(), false);
|
||||
ParsePredicateSuccessProj* parse_predicate_proj) {
|
||||
Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, parse_predicate_proj);
|
||||
IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, iff->Opcode(), false);
|
||||
_igvn.replace_input_of(if_proj->in(0), 1, bol);
|
||||
_igvn.replace_input_of(output_proj->in(0), 0, if_proj);
|
||||
set_idom(output_proj->in(0), if_proj, dom_depth(if_proj));
|
||||
_igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj);
|
||||
set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj));
|
||||
return if_proj;
|
||||
}
|
||||
|
||||
@ -446,9 +433,13 @@ void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealL
|
||||
IfProjNode*& iffast_pred,
|
||||
IfProjNode*& ifslow_pred) {
|
||||
if (predicate_block->has_parse_predicate()) {
|
||||
// We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will
|
||||
// be changed with the complete fix for Assertion Predicates.
|
||||
clone_parse_predicate_to_unswitched_loops(predicate_block, reason, iffast_pred, ifslow_pred);
|
||||
assert(iffast_pred->in(0)->is_ParsePredicate() && ifslow_pred->in(0)->is_ParsePredicate(),
|
||||
"must be success projections of the cloned Parse Predicates");
|
||||
clone_assertion_predicates_to_unswitched_loop(loop, old_new, reason, predicate_block->parse_predicate_success_proj(),
|
||||
iffast_pred, ifslow_pred);
|
||||
iffast_pred->as_IfTrue(), ifslow_pred->as_IfTrue());
|
||||
}
|
||||
}
|
||||
|
||||
@ -1156,7 +1147,7 @@ void PhaseIdealLoop::loop_predication_follow_branches(Node *n, IdealLoopTree *lo
|
||||
stack.push(in, 1);
|
||||
break;
|
||||
} else if (in->is_IfProj() &&
|
||||
in->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
|
||||
in->as_Proj()->is_uncommon_trap_if_pattern() &&
|
||||
(in->in(0)->Opcode() == Op_If ||
|
||||
in->in(0)->Opcode() == Op_RangeCheck)) {
|
||||
if (pf.to(in) * loop_trip_cnt >= 1) {
|
||||
@ -1305,19 +1296,20 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
|
||||
// them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub
|
||||
// loops are removed properly.
|
||||
IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
|
||||
IfProjNode* predicate_proj, IfProjNode* upper_bound_proj,
|
||||
int scale, Node* offset, Node* init, Node* limit, jint stride,
|
||||
ParsePredicateSuccessProj* parse_predicate_proj,
|
||||
IfProjNode* upper_bound_proj, const int scale, Node* offset,
|
||||
Node* init, Node* limit, const jint stride,
|
||||
Node* rng, bool& overflow, Deoptimization::DeoptReason reason) {
|
||||
// First predicate for the initial value on first loop iteration
|
||||
Node* opaque_init = new OpaqueLoopInitNode(C, init);
|
||||
register_new_node(opaque_init, upper_bound_proj);
|
||||
bool negate = (if_proj->_con != predicate_proj->_con);
|
||||
bool negate = (if_proj->_con != parse_predicate_proj->_con);
|
||||
BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng,
|
||||
(stride > 0) != (scale > 0), overflow);
|
||||
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over
|
||||
C->add_template_assertion_predicate_opaq(opaque_bol);
|
||||
register_new_node(opaque_bol, upper_bound_proj);
|
||||
IfProjNode* new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
|
||||
IfProjNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
|
||||
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
|
||||
assert(opaque_init->outcnt() > 0, "should be used");
|
||||
|
||||
@ -1333,14 +1325,14 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL
|
||||
register_new_node(max_value, new_proj);
|
||||
// init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi
|
||||
max_value = new CastIINode(max_value, loop->_head->as_CountedLoop()->phi()->bottom_type());
|
||||
register_new_node(max_value, predicate_proj);
|
||||
register_new_node(max_value, parse_predicate_proj);
|
||||
|
||||
bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0),
|
||||
overflow);
|
||||
opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1));
|
||||
C->add_template_assertion_predicate_opaq(opaque_bol);
|
||||
register_new_node(opaque_bol, new_proj);
|
||||
new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
|
||||
new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
|
||||
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
|
||||
assert(max_value->outcnt() > 0, "should be used");
|
||||
assert(assertion_predicate_has_loop_opaque_node(new_proj->in(0)->as_If()), "unexpected");
|
||||
@ -1350,13 +1342,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL
|
||||
|
||||
// Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for
|
||||
// range checks.
|
||||
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
|
||||
if (!UseLoopPredicate) return false;
|
||||
|
||||
if (!loop->_head->is_Loop()) {
|
||||
// Could be a simple region when irreducible loops are present.
|
||||
return false;
|
||||
}
|
||||
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree* loop) {
|
||||
LoopNode* head = loop->_head->as_Loop();
|
||||
|
||||
if (head->unique_ctrl_out()->is_NeverBranch()) {
|
||||
@ -1436,7 +1422,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
|
||||
IfProjNode* if_proj = n->as_IfProj();
|
||||
IfNode* iff = if_proj->in(0)->as_If();
|
||||
|
||||
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern();
|
||||
if (call == nullptr) {
|
||||
if (loop->is_loop_exit(iff)) {
|
||||
// stop processing the remaining projs in the list because the execution of them
|
||||
@ -1474,7 +1460,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
|
||||
while (if_proj_list.size() > 0) {
|
||||
Node* if_proj = if_proj_list.pop();
|
||||
float f = pf.to(if_proj);
|
||||
if (if_proj->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
|
||||
if (if_proj->as_Proj()->is_uncommon_trap_if_pattern() &&
|
||||
f * loop_trip_cnt >= 1) {
|
||||
ParsePredicateSuccessProj* profiled_loop_parse_predicate_proj =
|
||||
profiled_loop_predicate_block->parse_predicate_success_proj();
|
||||
@ -1546,7 +1532,7 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
|
||||
}
|
||||
|
||||
// self
|
||||
if (!_irreducible && !tail()->is_top()) {
|
||||
if (can_apply_loop_predication()) {
|
||||
hoisted |= phase->loop_predication_impl(this);
|
||||
}
|
||||
|
||||
@ -1556,3 +1542,7 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
|
||||
|
||||
return hoisted;
|
||||
}
|
||||
|
||||
bool IdealLoopTree::can_apply_loop_predication() {
|
||||
return _head->is_Loop() && !_irreducible && !tail()->is_top();
|
||||
}
|
||||
|
@ -177,7 +177,7 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) {
|
||||
// expensive nodes will notice the loop and skip over it to try to
|
||||
// move the node further up.
|
||||
if (ctl->is_CountedLoop() && ctl->in(1) != nullptr && ctl->in(1)->in(0) != nullptr && ctl->in(1)->in(0)->is_If()) {
|
||||
if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) {
|
||||
if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern()) {
|
||||
break;
|
||||
}
|
||||
next = idom(ctl->in(1)->in(0));
|
||||
@ -191,7 +191,7 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) {
|
||||
} else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != nullptr) {
|
||||
next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control();
|
||||
} else if (parent_ctl->is_If()) {
|
||||
if (!ctl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) {
|
||||
if (!ctl->as_Proj()->is_uncommon_trap_if_pattern()) {
|
||||
break;
|
||||
}
|
||||
assert(idom(ctl) == parent_ctl, "strange");
|
||||
@ -350,11 +350,6 @@ void PhaseIdealLoop::insert_loop_limit_check_predicate(ParsePredicateSuccessProj
|
||||
Deoptimization::Reason_loop_limit_check,
|
||||
Op_If);
|
||||
Node* iff = new_predicate_proj->in(0);
|
||||
assert(iff->Opcode() == Op_If, "bad graph shape");
|
||||
Node* conv = iff->in(1);
|
||||
assert(conv->Opcode() == Op_Conv2B, "bad graph shape");
|
||||
Node* opaq = conv->in(1);
|
||||
assert(opaq->Opcode() == Op_Opaque1, "bad graph shape");
|
||||
cmp_limit = _igvn.register_new_node_with_optimizer(cmp_limit);
|
||||
bol = _igvn.register_new_node_with_optimizer(bol);
|
||||
set_subtree_ctrl(bol, false);
|
||||
@ -564,19 +559,12 @@ Node* PhaseIdealLoop::loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv,
|
||||
void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop,
|
||||
SafePointNode* sfpt) {
|
||||
if (!C->too_many_traps(reason)) {
|
||||
Node* cont = _igvn.intcon(1);
|
||||
Node* opaq = new Opaque1Node(C, cont);
|
||||
_igvn.register_new_node_with_optimizer(opaq);
|
||||
Node* bol = new Conv2BNode(opaq);
|
||||
_igvn.register_new_node_with_optimizer(bol);
|
||||
set_subtree_ctrl(bol, false);
|
||||
ParsePredicateNode* iff = new ParsePredicateNode(inner_head->in(LoopNode::EntryControl), bol, reason);
|
||||
register_control(iff, loop, inner_head->in(LoopNode::EntryControl));
|
||||
Node* if_false = new IfFalseNode(iff);
|
||||
register_control(if_false, _ltree_root, iff);
|
||||
Node* if_true = new IfTrueNode(iff);
|
||||
register_control(if_true, loop, iff);
|
||||
C->add_parse_predicate_opaq(opaq);
|
||||
ParsePredicateNode* parse_predicate = new ParsePredicateNode(inner_head->in(LoopNode::EntryControl), reason, &_igvn);
|
||||
register_control(parse_predicate, loop, inner_head->in(LoopNode::EntryControl));
|
||||
Node* if_false = new IfFalseNode(parse_predicate);
|
||||
register_control(if_false, _ltree_root, parse_predicate);
|
||||
Node* if_true = new IfTrueNode(parse_predicate);
|
||||
register_control(if_true, loop, parse_predicate);
|
||||
|
||||
int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile);
|
||||
address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point();
|
||||
@ -1089,7 +1077,7 @@ int PhaseIdealLoop::extract_long_range_checks(const IdealLoopTree* loop, jlong s
|
||||
Node* c = loop->_body.at(i);
|
||||
if (c->is_IfProj() && c->in(0)->is_RangeCheck()) {
|
||||
IfProjNode* if_proj = c->as_IfProj();
|
||||
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern();
|
||||
if (call != nullptr) {
|
||||
Node* range = nullptr;
|
||||
Node* offset = nullptr;
|
||||
@ -4052,78 +4040,107 @@ void PhaseIdealLoop::log_loop_tree() {
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------collect_potentially_useful_predicates-----------------------
|
||||
// Helper function to collect potentially useful predicates to prevent them from
|
||||
// being eliminated by PhaseIdealLoop::eliminate_useless_predicates
|
||||
void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop, Unique_Node_List &useful_predicates) {
|
||||
if (loop->_child) { // child
|
||||
collect_potentially_useful_predicates(loop->_child, useful_predicates);
|
||||
// Eliminate all Parse and Template Assertion Predicates that are not associated with a loop anymore. The eliminated
|
||||
// predicates will be removed during the next round of IGVN.
|
||||
void PhaseIdealLoop::eliminate_useless_predicates() {
|
||||
if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) {
|
||||
return; // No predicates left.
|
||||
}
|
||||
|
||||
// self (only loops that we can apply loop predication may use their predicates)
|
||||
if (loop->_head->is_Loop() &&
|
||||
!loop->_irreducible &&
|
||||
!loop->tail()->is_top()) {
|
||||
LoopNode* lpn = loop->_head->as_Loop();
|
||||
Node* entry = lpn->in(LoopNode::EntryControl);
|
||||
const Predicates predicates(entry);
|
||||
const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
|
||||
if (loop_limit_check_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication
|
||||
IfProjNode* parse_predicate_proj = loop_limit_check_predicate_block->parse_predicate_success_proj();
|
||||
assert(parse_predicate_proj->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be");
|
||||
useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one
|
||||
}
|
||||
if (UseProfiledLoopPredicate) {
|
||||
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
|
||||
if (profiled_loop_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication
|
||||
IfProjNode* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj();
|
||||
useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one
|
||||
get_assertion_predicates(parse_predicate_proj, useful_predicates, true);
|
||||
}
|
||||
}
|
||||
eliminate_useless_parse_predicates();
|
||||
eliminate_useless_template_assertion_predicates();
|
||||
}
|
||||
|
||||
if (UseLoopPredicate) {
|
||||
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
|
||||
if (loop_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication
|
||||
IfProjNode* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj();
|
||||
useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one
|
||||
get_assertion_predicates(parse_predicate_proj, useful_predicates, true);
|
||||
}
|
||||
}
|
||||
// Eliminate all Parse Predicates that do not belong to a loop anymore by marking them useless. These will be removed
|
||||
// during the next round of IGVN.
|
||||
void PhaseIdealLoop::eliminate_useless_parse_predicates() {
|
||||
mark_all_parse_predicates_useless();
|
||||
if (C->has_loops()) {
|
||||
mark_loop_associated_parse_predicates_useful();
|
||||
}
|
||||
add_useless_parse_predicates_to_igvn_worklist();
|
||||
}
|
||||
|
||||
if (loop->_next) { // sibling
|
||||
collect_potentially_useful_predicates(loop->_next, useful_predicates);
|
||||
void PhaseIdealLoop::mark_all_parse_predicates_useless() const {
|
||||
for (int i = 0; i < C->parse_predicate_count(); i++) {
|
||||
C->parse_predicate(i)->mark_useless();
|
||||
}
|
||||
}
|
||||
|
||||
//------------------------eliminate_useless_predicates-----------------------------
|
||||
// Eliminate all inserted predicates if they could not be used by loop predication.
|
||||
// Note: it will also eliminates loop limits check predicate since it also uses
|
||||
// Opaque1 node (see Parse::add_predicate()).
|
||||
void PhaseIdealLoop::eliminate_useless_predicates() {
|
||||
if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) {
|
||||
return; // no predicate left
|
||||
void PhaseIdealLoop::mark_loop_associated_parse_predicates_useful() {
|
||||
for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) {
|
||||
IdealLoopTree* loop = iterator.current();
|
||||
if (loop->can_apply_loop_predication()) {
|
||||
mark_useful_parse_predicates_for_loop(loop);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Unique_Node_List useful_predicates; // to store useful predicates
|
||||
void PhaseIdealLoop::mark_useful_parse_predicates_for_loop(IdealLoopTree* loop) {
|
||||
Node* entry = loop->_head->in(LoopNode::EntryControl);
|
||||
const Predicates predicates(entry);
|
||||
ParsePredicateIterator iterator(predicates);
|
||||
while (iterator.has_next()) {
|
||||
iterator.next()->mark_useful();
|
||||
}
|
||||
}
|
||||
|
||||
void PhaseIdealLoop::add_useless_parse_predicates_to_igvn_worklist() {
|
||||
for (int i = 0; i < C->parse_predicate_count(); i++) {
|
||||
ParsePredicateNode* parse_predicate_node = C->parse_predicate(i);
|
||||
if (parse_predicate_node->is_useless()) {
|
||||
_igvn._worklist.push(parse_predicate_node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Eliminate all Template Assertion Predicates that do not belong to their originally associated loop anymore by
|
||||
// replacing the Opaque4 node of the If node with true. These nodes will be removed during the next round of IGVN.
|
||||
void PhaseIdealLoop::eliminate_useless_template_assertion_predicates() {
|
||||
Unique_Node_List useful_predicates;
|
||||
if (C->has_loops()) {
|
||||
collect_potentially_useful_predicates(_ltree_root->_child, useful_predicates);
|
||||
collect_useful_template_assertion_predicates(useful_predicates);
|
||||
}
|
||||
eliminate_useless_template_assertion_predicates(useful_predicates);
|
||||
}
|
||||
|
||||
void PhaseIdealLoop::collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates) {
|
||||
for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) {
|
||||
IdealLoopTree* loop = iterator.current();
|
||||
if (loop->can_apply_loop_predication()) {
|
||||
collect_useful_template_assertion_predicates_for_loop(loop, useful_predicates);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop,
|
||||
Unique_Node_List &useful_predicates) {
|
||||
Node* entry = loop->_head->in(LoopNode::EntryControl);
|
||||
const Predicates predicates(entry);
|
||||
if (UseProfiledLoopPredicate) {
|
||||
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
|
||||
if (profiled_loop_predicate_block->has_parse_predicate()) {
|
||||
IfProjNode* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj();
|
||||
get_assertion_predicates(parse_predicate_proj, useful_predicates, true);
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = C->parse_predicate_count(); i > 0; i--) {
|
||||
Node* n = C->parse_predicate_opaque1_node(i - 1);
|
||||
assert(n->Opcode() == Op_Opaque1, "must be");
|
||||
if (!useful_predicates.member(n)) { // not in the useful list
|
||||
_igvn.replace_node(n, n->in(1));
|
||||
}
|
||||
if (UseLoopPredicate) {
|
||||
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
|
||||
if (loop_predicate_block->has_parse_predicate()) {
|
||||
IfProjNode* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj();
|
||||
get_assertion_predicates(parse_predicate_proj, useful_predicates, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = C->template_assertion_predicate_count(); i > 0; i--) {
|
||||
Node* n = C->template_assertion_predicate_opaq_node(i - 1);
|
||||
assert(n->Opcode() == Op_Opaque4, "must be");
|
||||
if (!useful_predicates.member(n)) { // not in the useful list
|
||||
_igvn.replace_node(n, n->in(2));
|
||||
void PhaseIdealLoop::eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates) {
|
||||
for (int i = 0; i < C->template_assertion_predicate_count(); i++) {
|
||||
Node* opaque4 = C->template_assertion_predicate_opaq_node(i);
|
||||
assert(opaque4->Opcode() == Op_Opaque4, "must be");
|
||||
if (!useful_predicates.member(opaque4)) { // not in the useful list
|
||||
_igvn.replace_node(opaque4, opaque4->in(2));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -4561,7 +4578,7 @@ void PhaseIdealLoop::build_and_optimize() {
|
||||
}
|
||||
|
||||
// Perform loop predication before iteration splitting
|
||||
if (C->has_loops() && !C->major_progress() && (C->parse_predicate_count() > 0)) {
|
||||
if (UseLoopPredicate && C->has_loops() && !C->major_progress() && (C->parse_predicate_count() > 0)) {
|
||||
_ltree_root->_child->loop_predication(this);
|
||||
}
|
||||
|
||||
@ -4614,7 +4631,8 @@ void PhaseIdealLoop::build_and_optimize() {
|
||||
// until no more loop optimizations could be done.
|
||||
// After that switch predicates off and do more loop optimizations.
|
||||
if (!C->major_progress() && (C->parse_predicate_count() > 0)) {
|
||||
C->cleanup_parse_predicates(_igvn);
|
||||
C->mark_parse_predicate_nodes_useless(_igvn);
|
||||
assert(C->parse_predicate_count() == 0, "should be zero now");
|
||||
if (TraceLoopOpts) {
|
||||
tty->print_cr("PredicatesOff");
|
||||
}
|
||||
@ -6096,7 +6114,7 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) {
|
||||
if (!new_ctrl->is_Proj()) {
|
||||
break;
|
||||
}
|
||||
CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern();
|
||||
if (call == nullptr) {
|
||||
break;
|
||||
}
|
||||
|
@ -646,6 +646,7 @@ public:
|
||||
// Perform optimization to use the loop predicates for null checks and range checks.
|
||||
// Applies to any loop level (not just the innermost one)
|
||||
bool loop_predication( PhaseIdealLoop *phase);
|
||||
bool can_apply_loop_predication();
|
||||
|
||||
// Perform iteration-splitting on inner loops. Split iterations to
|
||||
// avoid range checks or one-shot null checks. Returns false if the
|
||||
@ -1330,8 +1331,9 @@ public:
|
||||
bool* p_short_scale, int depth);
|
||||
|
||||
// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
|
||||
IfProjNode* create_new_if_for_predicate(IfProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason,
|
||||
int opcode, bool rewire_uncommon_proj_phi_inputs = false);
|
||||
IfProjNode* create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
|
||||
Deoptimization::DeoptReason reason, int opcode,
|
||||
bool rewire_uncommon_proj_phi_inputs = false);
|
||||
|
||||
private:
|
||||
// Helper functions for create_new_if_for_predicate()
|
||||
@ -1362,15 +1364,27 @@ public:
|
||||
void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt,
|
||||
PathFrequency& pf, Node_Stack& stack, VectorSet& seen,
|
||||
Node_List& if_proj_list);
|
||||
IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, IfProjNode* predicate_proj,
|
||||
IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
|
||||
ParsePredicateSuccessProj* parse_predicate_proj,
|
||||
IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit,
|
||||
jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason);
|
||||
Node* add_range_check_elimination_assertion_predicate(IdealLoopTree* loop, Node* predicate_proj, int scale_con,
|
||||
Node* offset, Node* limit, jint stride_con, Node* value);
|
||||
|
||||
// Helper function to collect predicate for eliminating the useless ones
|
||||
void collect_potentially_useful_predicates(IdealLoopTree *loop, Unique_Node_List &predicate_opaque1);
|
||||
void eliminate_useless_predicates();
|
||||
|
||||
void eliminate_useless_parse_predicates();
|
||||
void mark_all_parse_predicates_useless() const;
|
||||
void mark_loop_associated_parse_predicates_useful();
|
||||
static void mark_useful_parse_predicates_for_loop(IdealLoopTree* loop);
|
||||
void add_useless_parse_predicates_to_igvn_worklist();
|
||||
|
||||
void eliminate_useless_template_assertion_predicates();
|
||||
void collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates);
|
||||
static void collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, Unique_Node_List& useful_predicates);
|
||||
void eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates);
|
||||
|
||||
void eliminate_useless_zero_trip_guard();
|
||||
|
||||
bool has_control_dependencies_from_predicates(LoopNode* head) const;
|
||||
@ -1621,14 +1635,15 @@ private:
|
||||
IfProjNode*& ifslow_pred);
|
||||
void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason,
|
||||
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
|
||||
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* predicate_proj, Node* new_entry,
|
||||
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
|
||||
Deoptimization::DeoptReason reason, bool slow_loop);
|
||||
void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
|
||||
Deoptimization::DeoptReason reason, IfProjNode* old_predicate_proj,
|
||||
IfProjNode* iffast_pred, IfProjNode* ifslow_pred);
|
||||
ParsePredicateSuccessProj* fast_loop_parse_predicate_proj,
|
||||
ParsePredicateSuccessProj* slow_loop_parse_predicate_proj);
|
||||
IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
|
||||
Deoptimization::DeoptReason reason,
|
||||
IfProjNode* output_proj);
|
||||
ParsePredicateSuccessProj* parse_predicate_proj);
|
||||
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;
|
||||
|
||||
bool _created_loop_node;
|
||||
|
@ -2068,7 +2068,7 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new,
|
||||
// make sure the Bool/Cmp input is cloned down to avoid a Phi between
|
||||
// the AllocateArray node and its ValidLengthTest input that could cause
|
||||
// split if to break.
|
||||
if (use->is_If() || use->is_CMove() || C->is_predicate_opaq(use) || use->Opcode() == Op_Opaque4 ||
|
||||
if (use->is_If() || use->is_CMove() || use->Opcode() == Op_Opaque4 ||
|
||||
(use->Opcode() == Op_AllocateArray && use->in(AllocateNode::ValidLengthTest) == old)) {
|
||||
// Since this code is highly unlikely, we lazily build the worklist
|
||||
// of such Nodes to go split.
|
||||
|
@ -537,7 +537,7 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType
|
||||
} else if (mem->is_ArrayCopy()) {
|
||||
Node* ctl = mem->in(0);
|
||||
Node* m = mem->in(TypeFunc::Memory);
|
||||
if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
|
||||
if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj()) {
|
||||
// pin the loads in the uncommon trap path
|
||||
ctl = sfpt_ctl;
|
||||
m = sfpt_mem;
|
||||
|
@ -184,9 +184,9 @@ uint ProjNode::ideal_reg() const {
|
||||
//-------------------------------is_uncommon_trap_proj----------------------------
|
||||
// Return uncommon trap call node if proj is for "proj->[region->..]call_uct"
|
||||
// null otherwise
|
||||
CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason reason) {
|
||||
int path_limit = 10;
|
||||
Node* out = this;
|
||||
CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason reason) const {
|
||||
const int path_limit = 10;
|
||||
const Node* out = this;
|
||||
for (int ct = 0; ct < path_limit; ct++) {
|
||||
out = out->unique_ctrl_out_or_null();
|
||||
if (out == nullptr)
|
||||
@ -213,31 +213,14 @@ CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason
|
||||
// |
|
||||
// V
|
||||
// other_proj->[region->..]call_uct"
|
||||
// null otherwise
|
||||
// "must_reason_predicate" means the uct reason must be Reason_predicate
|
||||
CallStaticJavaNode* ProjNode::is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason) {
|
||||
Node *in0 = in(0);
|
||||
if (!in0->is_If()) return nullptr;
|
||||
// Variation of a dead If node.
|
||||
if (in0->outcnt() < 2) return nullptr;
|
||||
IfNode* iff = in0->as_If();
|
||||
|
||||
// we need "If(Conv2B(Opaque1(...)))" pattern for reason_predicate
|
||||
if (reason != Deoptimization::Reason_none) {
|
||||
if (iff->in(1)->Opcode() != Op_Conv2B ||
|
||||
iff->in(1)->in(1)->Opcode() != Op_Opaque1) {
|
||||
return nullptr;
|
||||
}
|
||||
// or null otherwise.
|
||||
CallStaticJavaNode* ProjNode::is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason) const {
|
||||
Node* iff = in(0);
|
||||
if (!iff->is_If() || iff->outcnt() < 2) {
|
||||
// Not a projection of an If or variation of a dead If node.
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ProjNode* other_proj = iff->proj_out(1-_con);
|
||||
CallStaticJavaNode* call = other_proj->is_uncommon_trap_proj(reason);
|
||||
if (call != nullptr) {
|
||||
assert(reason == Deoptimization::Reason_none ||
|
||||
Compile::current()->is_predicate_opaq(iff->in(1)->in(1)), "should be on the list");
|
||||
return call;
|
||||
}
|
||||
return nullptr;
|
||||
return other_if_proj()->is_uncommon_trap_proj(reason);
|
||||
}
|
||||
|
||||
ProjNode* ProjNode::other_if_proj() const {
|
||||
|
@ -93,13 +93,13 @@ public:
|
||||
|
||||
// Return uncommon trap call node if proj is for "proj->[region->..]call_uct"
|
||||
// null otherwise
|
||||
CallStaticJavaNode* is_uncommon_trap_proj(Deoptimization::DeoptReason reason);
|
||||
CallStaticJavaNode* is_uncommon_trap_proj(Deoptimization::DeoptReason reason = Deoptimization::Reason_none) const;
|
||||
// Return uncommon trap call node for "if(test)-> proj -> ...
|
||||
// |
|
||||
// V
|
||||
// other_proj->[region->..]call_uct"
|
||||
// null otherwise
|
||||
CallStaticJavaNode* is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason);
|
||||
CallStaticJavaNode* is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason = Deoptimization::Reason_none) const;
|
||||
|
||||
// Return other proj node when this is a If proj node
|
||||
ProjNode* other_if_proj() const;
|
||||
|
@ -508,6 +508,10 @@ Node *Node::clone() const {
|
||||
// If it is applicable, it will happen anyway when the cloned node is registered with IGVN.
|
||||
n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
|
||||
}
|
||||
if (n->is_ParsePredicate()) {
|
||||
C->add_parse_predicate(n->as_ParsePredicate());
|
||||
}
|
||||
|
||||
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
|
||||
bs->register_potential_barrier_node(n);
|
||||
|
||||
@ -609,6 +613,9 @@ void Node::destruct(PhaseValues* phase) {
|
||||
if (Opcode() == Op_Opaque4) {
|
||||
compile->remove_template_assertion_predicate_opaq(this);
|
||||
}
|
||||
if (is_ParsePredicate()) {
|
||||
compile->remove_parse_predicate(as_ParsePredicate());
|
||||
}
|
||||
if (for_post_loop_opts_igvn()) {
|
||||
compile->remove_from_post_loop_opts_igvn(this);
|
||||
}
|
||||
|
@ -69,7 +69,7 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p
|
||||
}
|
||||
|
||||
Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) {
|
||||
CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
|
||||
CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern();
|
||||
if (uct_call == nullptr) {
|
||||
return Deoptimization::Reason_none;
|
||||
}
|
||||
@ -84,6 +84,30 @@ bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason d
|
||||
}
|
||||
}
|
||||
|
||||
ParsePredicateIterator::ParsePredicateIterator(const Predicates& predicates) : _current_index(0) {
|
||||
const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
|
||||
if (loop_limit_check_predicate_block->has_parse_predicate()) {
|
||||
_parse_predicates.push(loop_limit_check_predicate_block->parse_predicate());
|
||||
}
|
||||
if (UseProfiledLoopPredicate) {
|
||||
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
|
||||
if (profiled_loop_predicate_block->has_parse_predicate()) {
|
||||
_parse_predicates.push(profiled_loop_predicate_block->parse_predicate());
|
||||
}
|
||||
}
|
||||
if (UseLoopPredicate) {
|
||||
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
|
||||
if (loop_predicate_block->has_parse_predicate()) {
|
||||
_parse_predicates.push(loop_predicate_block->parse_predicate());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ParsePredicateNode* ParsePredicateIterator::next() {
|
||||
assert(has_next(), "always check has_next() first");
|
||||
return _parse_predicates.at(_current_index++);
|
||||
}
|
||||
|
||||
// Walk over all Regular Predicates of this block (if any) and return the first node not belonging to the block
|
||||
// anymore (i.e. entry to the first Regular Predicate in this block if any or `regular_predicate_proj` otherwise).
|
||||
Node* PredicateBlock::skip_regular_predicates(Node* regular_predicate_proj, Deoptimization::DeoptReason deopt_reason) {
|
||||
|
@ -355,4 +355,19 @@ class Predicates : public StackObj {
|
||||
return _entry != _loop_entry;
|
||||
}
|
||||
};
|
||||
|
||||
// This class iterates over the Parse Predicates of a loop.
|
||||
class ParsePredicateIterator : public StackObj {
|
||||
GrowableArray<ParsePredicateNode*> _parse_predicates;
|
||||
int _current_index;
|
||||
|
||||
public:
|
||||
ParsePredicateIterator(const Predicates& predicates);
|
||||
|
||||
bool has_next() const {
|
||||
return _current_index < _parse_predicates.length();
|
||||
}
|
||||
|
||||
ParsePredicateNode* next();
|
||||
};
|
||||
#endif // SHARE_OPTO_PREDICATES_HPP
|
||||
|
Loading…
x
Reference in New Issue
Block a user