8324809: compiler can crash with SOE while proving if two recursive types are disjoint

Reviewed-by: mcimadamore

src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java

@@ -4734,6 +4734,10 @@ public class Types {
 
         boolean high;
         boolean rewriteTypeVars;
+        // map to avoid visiting same type argument twice, like in Foo<T>.Bar<T>
+        Map<Type, Type> argMap = new HashMap<>();
+        // cycle detection within an argument, see JDK-8324809
+        Set<Type> seen = new HashSet<>();
 
         Rewriter(boolean high, boolean rewriteTypeVars) {
             this.high = high;
@@ -4745,7 +4749,10 @@ public class Types {
             ListBuffer<Type> rewritten = new ListBuffer<>();
             boolean changed = false;
             for (Type arg : t.allparams()) {
-                Type bound = visit(arg);
+                Type bound = argMap.get(arg);
+                if (bound == null) {
+                    argMap.put(arg, bound = visit(arg));
+                }
                 if (arg != bound) {
                     changed = true;
                 }
@@ -4774,13 +4781,17 @@ public class Types {
 
         @Override
         public Type visitTypeVar(TypeVar t, Void s) {
-            if (rewriteTypeVars) {
+            if (seen.add(t)) {
-                Type bound = t.getUpperBound().contains(t) ?
+                if (rewriteTypeVars) {
-                        erasure(t.getUpperBound()) :
+                    Type bound = t.getUpperBound().contains(t) ?
-                        visit(t.getUpperBound());
+                            erasure(t.getUpperBound()) :
-                return rewriteAsWildcardType(bound, t, EXTENDS);
+                            visit(t.getUpperBound());
+                    return rewriteAsWildcardType(bound, t, EXTENDS);
+                } else {
+                    return t;
+                }
             } else {
-                return t;
+                return rewriteTypeVars ? makeExtendsWildcard(syms.objectType, t) : t;
             }
         }
 

test/langtools/tools/javac/generics/wildcards/SOEWhileProvingDisjointnessTest.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8324809
+ * @summary compiler can crash with SOE while proving if two recursive types are disjoint
+ * @compile SOEWhileProvingDisjointnessTest.java
+ */
+
+class SOEWhileProvingDisjointnessTest {
+    class Criteria<B extends Builder<? extends Criteria>> {
+        public <D extends Builder<E>, E extends Criteria<D>> D builder() {
+            return (D) new Builder<>();
+        }
+    }
+
+    class Builder<C extends Criteria<? extends Builder<C>>> {}
+}