8179950: Custom system class loader using Enum.valueOf in its initialization triggers java.lang.InternalError

Reviewed-by: alanb, dfuchs, lancea

jdk/src/java.base/share/classes/jdk/internal/reflect/Reflection.java

@@ -31,6 +31,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import jdk.internal.HotSpotIntrinsicCandidate;
+import jdk.internal.loader.ClassLoaders;
 import jdk.internal.misc.VM;
 
 /** Common utility routines used by both java.lang and
@@ -315,23 +316,13 @@ public class Reflection {
      */
     public static boolean isCallerSensitive(Method m) {
         final ClassLoader loader = m.getDeclaringClass().getClassLoader();
-        if (VM.isSystemDomainLoader(loader) || isExtClassLoader(loader))  {
+        if (VM.isSystemDomainLoader(loader) ||
+                loader == ClassLoaders.platformClassLoader()) {
             return m.isAnnotationPresent(CallerSensitive.class);
         }
         return false;
     }
 
-    private static boolean isExtClassLoader(ClassLoader loader) {
-        ClassLoader cl = ClassLoader.getSystemClassLoader();
-        while (cl != null) {
-            if (cl.getParent() == null && cl == loader) {
-                return true;
-            }
-            cl = cl.getParent();
-        }
-        return false;
-    }
-
     /**
      * Returns an IllegalAccessException with an exception message based on
      * the access that is denied.

jdk/test/java/lang/ClassLoader/CustomSystemLoader/CustomLoader.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.PrintStream;
+
+/*
+ * Custom system class loader.
+ */
+public class CustomLoader extends ClassLoader {
+    private static PrintStream out = System.out;
+    public  static ClassLoader INSTANCE;
+
+    public CustomLoader(ClassLoader classLoader) {
+        super("CustomSystemLoader", classLoader);
+        assert INSTANCE == null;
+        INSTANCE = this;
+
+        // test cases to validate that ClassLoader::getSystemClassLoader
+        // is not triggered during custom system class loader initialization
+        testEnumValueOf();
+    }
+
+    static void testEnumValueOf() {
+        TestEnum e = java.lang.Enum.valueOf(TestEnum.class, "C1");
+        if (e != TestEnum.C1) {
+            throw new RuntimeException("Expected: " + TestEnum.C1 + " got: " + e);
+        }
+    }
+
+    @Override
+    public Class<?> loadClass(String name) throws ClassNotFoundException {
+        out.println("CustomLoader: loading class: " + name);
+        return super.loadClass(name);
+    }
+
+    static enum TestEnum {
+        C1, C2, C3
+    }
+}

jdk/test/java/lang/ClassLoader/CustomSystemLoader/InitSystemLoaderTest.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8179950
+ * @build CustomLoader InitSystemLoaderTest
+ * @run main/othervm -Djava.system.class.loader=CustomLoader InitSystemLoaderTest
+ * @summary Test custom system loader initialization and verify their ancestors
+ */
+
+public class InitSystemLoaderTest {
+    public static void main(String... args) {
+        // check that system class loader is the custom loader
+        ClassLoader loader = ClassLoader.getSystemClassLoader();
+        if (loader != CustomLoader.INSTANCE) {
+            throw new RuntimeException("Expected custom loader: "
+                + CustomLoader.INSTANCE + " got: " + loader);
+        }
+
+        // parent of the custom loader should be builtin system class loader
+        ClassLoader builtinSystemLoader = loader.getParent();
+        ClassLoader grandparent = builtinSystemLoader.getParent();
+        if (grandparent != ClassLoader.getPlatformClassLoader()) {
+            throw new RuntimeException("Expected class loader ancestor: "
+                + ClassLoader.getPlatformClassLoader() + " got: " + grandparent);
+        }
+    }
+}