8201368: IfNode::fold_compares() may lead to incorrect execution

Reviewed-by: neliasso, kvn

src/hotspot/share/opto/cfgnode.hpp

@@ -298,6 +298,7 @@ private:
   void reroute_side_effect_free_unc(ProjNode* proj, ProjNode* dom_proj, PhaseIterGVN* igvn);
   ProjNode* uncommon_trap_proj(CallStaticJavaNode*& call) const;
   bool fold_compares_helper(ProjNode* proj, ProjNode* success, ProjNode* fail, PhaseIterGVN* igvn);
+  static bool is_dominator_unc(CallStaticJavaNode* dom_unc, CallStaticJavaNode* unc);
 
 protected:
   ProjNode* range_check_trap_proj(int& flip, Node*& l, Node*& r);

src/hotspot/share/opto/ifnode.cpp

@@ -775,6 +775,38 @@ bool IfNode::has_shared_region(ProjNode* proj, ProjNode*& success, ProjNode*& fa
   return success != NULL && fail != NULL;
 }
 
+bool IfNode::is_dominator_unc(CallStaticJavaNode* dom_unc, CallStaticJavaNode* unc) {
+  // Different methods and methods containing jsrs are not supported.
+  ciMethod* method = unc->jvms()->method();
+  ciMethod* dom_method = dom_unc->jvms()->method();
+  if (method != dom_method || method->has_jsrs()) {
+    return false;
+  }
+  // Check that both traps are in the same activation of the method (instead
+  // of two activations being inlined through different call sites) by verifying
+  // that the call stacks are equal for both JVMStates.
+  JVMState* dom_caller = dom_unc->jvms()->caller();
+  JVMState* caller = unc->jvms()->caller();
+  if ((dom_caller == NULL) != (caller == NULL)) {
+    // The current method must either be inlined into both dom_caller and
+    // caller or must not be inlined at all (top method). Bail out otherwise.
+    return false;
+  } else if (dom_caller != NULL && !dom_caller->same_calls_as(caller)) {
+    return false;
+  }
+  // Check that the bci of the dominating uncommon trap dominates the bci
+  // of the dominated uncommon trap. Otherwise we may not re-execute
+  // the dominated check after deoptimization from the merged uncommon trap.
+  ciTypeFlow* flow = dom_method->get_flow_analysis();
+  int bci = unc->jvms()->bci();
+  int dom_bci = dom_unc->jvms()->bci();
+  if (!flow->is_dominated_by(bci, dom_bci)) {
+    return false;
+  }
+
+  return true;
+}
+
 // Return projection that leads to an uncommon trap if any
 ProjNode* IfNode::uncommon_trap_proj(CallStaticJavaNode*& call) const {
   for (int i = 0; i < 2; i++) {
@@ -811,31 +843,7 @@ bool IfNode::has_only_uncommon_traps(ProjNode* proj, ProjNode*& success, ProjNod
         return false;
       }
 
-      // Different methods and methods containing jsrs are not supported.
+      if (!is_dominator_unc(dom_unc, unc)) {
-      ciMethod* method = unc->jvms()->method();
-      ciMethod* dom_method = dom_unc->jvms()->method();
-      if (method != dom_method || method->has_jsrs()) {
-        return false;
-      }
-      // Check that both traps are in the same activation of the method (instead
-      // of two activations being inlined through different call sites) by verifying
-      // that the call stacks are equal for both JVMStates.
-      JVMState* dom_caller = dom_unc->jvms()->caller();
-      JVMState* caller = unc->jvms()->caller();
-      if ((dom_caller == NULL) != (caller == NULL)) {
-        // The current method must either be inlined into both dom_caller and
-        // caller or must not be inlined at all (top method). Bail out otherwise.
-        return false;
-      } else if (dom_caller != NULL && !dom_caller->same_calls_as(caller)) {
-        return false;
-      }
-      // Check that the bci of the dominating uncommon trap dominates the bci
-      // of the dominated uncommon trap. Otherwise we may not re-execute
-      // the dominated check after deoptimization from the merged uncommon trap.
-      ciTypeFlow* flow = dom_method->get_flow_analysis();
-      int bci = unc->jvms()->bci();
-      int dom_bci = dom_unc->jvms()->bci();
-      if (!flow->is_dominated_by(bci, dom_bci)) {
         return false;
       }
 
@@ -843,6 +851,8 @@ bool IfNode::has_only_uncommon_traps(ProjNode* proj, ProjNode*& success, ProjNod
       // will be changed and the state of the dominating If will be
       // used. Checked that we didn't apply this transformation in a
       // previous compilation and it didn't cause too many traps
+      ciMethod* dom_method = dom_unc->jvms()->method();
+      int dom_bci = dom_unc->jvms()->bci();
       if (!igvn->C->too_many_traps(dom_method, dom_bci, Deoptimization::Reason_unstable_fused_if) &&
           !igvn->C->too_many_traps(dom_method, dom_bci, Deoptimization::Reason_range_check)) {
         success = unc_proj;
@@ -1220,6 +1230,10 @@ bool IfNode::is_side_effect_free_test(ProjNode* proj, PhaseIterGVN* igvn) {
         return false;
       }
 
+      if (!is_dominator_unc(dom_unc, unc)) {
+        return false;
+      }
+
       return true;
     }
   }

test/hotspot/jtreg/compiler/uncommontrap/FoldedIfNonDomMidIf.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8201368
+ * @summary IfNode::fold_compares() may lead to incorrect execution
+ *
+ * @run main/othervm -XX:-TieredCompilation -XX:-UseOnStackReplacement -XX:-BackgroundCompilation FoldedIfNonDomMidIf
+ *
+ */
+
+public class FoldedIfNonDomMidIf {
+    public static void main(String[] args) {
+        for (int i = 0; i < 20_000; i++) {
+            test_helper(0, 0);
+            test_helper(20, 0);
+            test(12);
+        }
+        if (test(14) != null) {
+            throw new RuntimeException("Incorrect code execution");
+        }
+    }
+
+    private static Object test(int i) {
+        return test_helper(i, 0x42);
+    }
+
+    static class A {
+
+    }
+
+    static final MyException myex = new MyException();
+
+    private static Object test_helper(int i, int j) {
+        Object res = null;
+        try {
+            if (i < 10) {
+                throw myex;
+            }
+
+            if (i == 14) {
+
+            }
+
+            if (i > 15) {
+                throw myex;
+            }
+        } catch (MyException e) {
+            if (j == 0x42) {
+                res = new A();
+            }
+        }
+        return res;
+    }
+
+    private static class MyException extends Exception {
+    }
+}