8256934: C2: assert(C->live_nodes() <= C->max_node_limit()) failed: Live Node limit exceeded limit

Reviewed-by: roland, vlivanov

src/hotspot/share/opto/loopopts.cpp

@@ -2819,6 +2819,12 @@ int PhaseIdealLoop::clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, No
       worklist.push(use);
     }
   }
+
+  if (C->check_node_count(worklist.size() + NodeLimitFudgeFactor,
+                          "Too many clones required in clone_for_use_outside_loop in partial peeling")) {
+    return -1;
+  }
+
   while( worklist.size() ) {
     Node *use = worklist.pop();
     if (!has_node(use) || use->in(0) == C->top()) continue;
@@ -3371,6 +3377,7 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
 #endif
 
   // Evacuate nodes in peel region into the not_peeled region if possible
+  bool too_many_clones = false;
   uint new_phi_cnt = 0;
   uint cloned_for_outside_use = 0;
   for (uint i = 0; i < peel_list.size();) {
@@ -3387,7 +3394,12 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
           // if not pinned and not a load (which maybe anti-dependent on a store)
           // and not a CMove (Matcher expects only bool->cmove).
           if (n->in(0) == NULL && !n->is_Load() && !n->is_CMove()) {
-            cloned_for_outside_use += clone_for_use_outside_loop(loop, n, worklist);
+            int new_clones = clone_for_use_outside_loop(loop, n, worklist);
+            if (new_clones == -1) {
+              too_many_clones = true;
+              break;
+            }
+            cloned_for_outside_use += new_clones;
             sink_list.push(n);
             peel.remove(n->_idx);
             not_peel.set(n->_idx);
@@ -3415,9 +3427,9 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
   bool exceed_node_budget = !may_require_nodes(estimate);
   bool exceed_phi_limit = new_phi_cnt > old_phi_cnt + PartialPeelNewPhiDelta;
 
-  if (exceed_node_budget || exceed_phi_limit) {
+  if (too_many_clones || exceed_node_budget || exceed_phi_limit) {
 #ifndef PRODUCT
-    if (TracePartialPeeling) {
+    if (TracePartialPeeling && exceed_phi_limit) {
       tty->print_cr("\nToo many new phis: %d  old %d new cmpi: %c",
                     new_phi_cnt, old_phi_cnt, new_peel_if != NULL?'T':'F');
     }

test/hotspot/jtreg/compiler/loopopts/TestPartialPeelingSinkNodes.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @bug 8256934
+ * @summary Sinking of nodes in partial peeling creates too many clones resulting in a live node limit exceeded assertion failure.
+ * @requires vm.compiler2.enabled
+ * @run main/othervm -Xcomp -XX:-TieredCompilation -XX:CompileCommand=compileonly,compiler.loopopts.TestPartialPeelingSinkNodes::test
+ *                   compiler.loopopts.TestPartialPeelingSinkNodes
+ */
+
+package compiler.loopopts;
+
+public class TestPartialPeelingSinkNodes {
+    static int i5 = 168, iFld = 2, x, y;
+    static boolean b = false, b2 = false;
+
+    public static void main(String[] strArr) {
+        test();
+    }
+
+    // The algorithm in partial peeling creates ~90000 nodes for this method which triggers the assertion failure.
+    public static void test() {
+        for (int i = 0; i < 2480; i++) {
+            int i2 = -37052, i3 = 39651, i4 = -37052;
+            int i5 = 168, i6 = -133, i7 = 1, i8 = -10;
+            double d = -20.82293;
+
+            float fArr[] = new float[400];
+            for (int j = 0; j < 400; j++) {
+                fArr[j] = (j % 2 == 0) ? 0.300F + j : 0.300F - j;
+            }
+
+            while (--i5 > 0) {
+                i6 = 1;
+                do {
+                    i4 += (((i6 * i2) + i3) - i3);
+                    i2 += i4;
+                } while (++i6 < 9);
+                i3 -= i4;
+                for (i7 = 1; i7 < 18; i7++) {
+                    i4 = i5;
+                    d -= i4;
+                    i2 -= i8;
+                    i2 = i8;
+                }
+            }
+        }
+    }
+}
+