8207355: C1 compilation hangs in ComputeLinearScanOrder::compute_dominator

Bail out from recursive computation if the block was already processed.

Reviewed-by: kvn, iveresov

src/hotspot/share/c1/c1_IR.cpp

@@ -497,6 +497,7 @@ class ComputeLinearScanOrder : public StackObj {
   // computation of final block order
   BlockBegin* common_dominator(BlockBegin* a, BlockBegin* b);
   void compute_dominator(BlockBegin* cur, BlockBegin* parent);
+  void compute_dominator_impl(BlockBegin* cur, BlockBegin* parent);
   int  compute_weight(BlockBegin* cur);
   bool ready_for_processing(BlockBegin* cur);
   void sort_into_work_list(BlockBegin* b);
@@ -770,6 +771,14 @@ BlockBegin* ComputeLinearScanOrder::common_dominator(BlockBegin* a, BlockBegin*
 }
 
 void ComputeLinearScanOrder::compute_dominator(BlockBegin* cur, BlockBegin* parent) {
+  init_visited();
+  compute_dominator_impl(cur, parent);
+}
+
+void ComputeLinearScanOrder::compute_dominator_impl(BlockBegin* cur, BlockBegin* parent) {
+  // Mark as visited to avoid recursive calls with same parent
+  set_visited(cur);
+
   if (cur->dominator() == NULL) {
     TRACE_LINEAR_SCAN(4, tty->print_cr("DOM: initializing dominator of B%d to B%d", cur->block_id(), parent->block_id()));
     cur->set_dominator(parent);
@@ -788,7 +797,9 @@ void ComputeLinearScanOrder::compute_dominator(BlockBegin* cur, BlockBegin* pare
   int num_cur_xhandler = cur->number_of_exception_handlers();
   for (int j = 0; j < num_cur_xhandler; j++) {
     BlockBegin* xhandler = cur->exception_handler_at(j);
-    compute_dominator(xhandler, parent);
+    if (!is_visited(xhandler)) {
+      compute_dominator_impl(xhandler, parent);
+    }
   }
 }
 

test/hotspot/jtreg/compiler/c1/TestLinearScanOrderMain.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8207355
+ * @compile TestLinearScanOrder.jasm
+ * @run main/othervm -Xcomp -XX:+TieredCompilation -XX:TieredStopAtLevel=1
+ *                   -XX:CompileCommand=compileonly,compiler.c1.TestLinearScanOrder::test
+ *                   compiler.c1.TestLinearScanOrderMain
+ */
+
+package compiler.c1;
+
+// WARNING: This test will not fail but time out in C1's ComputeLinearScanOrder phase
+public class TestLinearScanOrderMain {
+    public static void main(String[] args) {
+        if (TestLinearScanOrder.test() != 42) {
+            throw new RuntimeException("Test failed");
+        }
+    }
+}