8253353: Crash in C2: guarantee(n != NULL) failed: No Node

Co-authored-by: Fei Yang <fyang@openjdk.org>
Reviewed-by: vlivanov, neliasso

src/hotspot/share/opto/loopnode.cpp

@@ -291,6 +291,7 @@ IdealLoopTree* PhaseIdealLoop::insert_outer_loop(IdealLoopTree* loop, LoopNode*
   loop->_parent = outer_ilt;
   loop->_next = NULL;
   loop->_nest++;
+  assert(loop->_nest <= SHRT_MAX, "sanity");
   return outer_ilt;
 }
 
@@ -2614,6 +2615,7 @@ bool IdealLoopTree::is_member(const IdealLoopTree *l) const {
 //------------------------------set_nest---------------------------------------
 // Set loop tree nesting depth.  Accumulate _has_call bits.
 int IdealLoopTree::set_nest( uint depth ) {
+  assert(depth <= SHRT_MAX, "sanity");
   _nest = depth;
   int bits = _has_call;
   if( _child ) bits |= _child->set_nest(depth+1);

src/hotspot/share/opto/loopnode.hpp

@@ -611,7 +611,7 @@ public:
 
   Node_List _body;              // Loop body for inner loops
 
-  uint8_t _nest;                // Nesting depth
+  uint16_t _nest;               // Nesting depth
   uint8_t _irreducible:1,       // True if irreducible
           _has_call:1,          // True if has call safepoint
           _has_sfpt:1,          // True if has non-call safepoint

test/hotspot/jtreg/compiler/loopopts/TestNestedIrreducibleLoopsMain.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8253353
+ * @summary Tests custom bytecode with deep nested irreducible loops.
+ *
+ * @compile TestNestedIrreducibleLoops.jasm
+ * @run main/othervm -Xbatch -XX:CompileCommand=dontinline,TestNestedIrreducibleLoops::*
+ *                   -XX:CompileCommand=exclude,TestNestedIrreducibleLoopsMain::main
+ *                   TestNestedIrreducibleLoopsMain
+ */
+
+public class TestNestedIrreducibleLoopsMain {
+    public static void main(String[] args) {
+        TestNestedIrreducibleLoops t = new TestNestedIrreducibleLoops();
+        t.loopCounter = 3;
+        int j;
+        for (int i = 0; i < 11000; i++) {
+            t.start = i & 0x3ff;
+            j = t.test(); // Produces deep nested irreducible loops
+        }
+    }
+}