8250219: Proxy::newProxyInstance spec should specify the behavior if a given proxy interface is hidden

Reviewed-by: alanb

src/java.base/share/classes/java/lang/reflect/Proxy.java

@@ -674,12 +674,6 @@ public class Proxy implements java.io.Serializable {
         {
             Map<Class<?>, Boolean> interfaceSet = new IdentityHashMap<>(interfaces.size());
             for (Class<?> intf : interfaces) {
-                /*
-                 * Verify that the class loader resolves the name of this
-                 * interface to the same Class object.
-                 */
-                ensureVisible(loader, intf);
-
                 /*
                  * Verify that the Class object actually represents an
                  * interface.
@@ -688,6 +682,16 @@ public class Proxy implements java.io.Serializable {
                     throw new IllegalArgumentException(intf.getName() + " is not an interface");
                 }
 
+                if (intf.isHidden()) {
+                    throw new IllegalArgumentException(intf.getName() + " is a hidden interface");
+                }
+
+                /*
+                 * Verify that the class loader resolves the name of this
+                 * interface to the same Class object.
+                 */
+                ensureVisible(loader, intf);
+
                 /*
                  * Verify that this interface is not a duplicate.
                  */
@@ -905,7 +909,8 @@ public class Proxy implements java.io.Serializable {
      * if any of the following restrictions is violated:</a>
      * <ul>
      * <li>All of {@code Class} objects in the given {@code interfaces} array
-     * must represent interfaces, not classes or primitive types.
+     * must represent {@linkplain Class#isHidden() non-hidden} interfaces,
+     * not classes or primitive types.
      *
      * <li>No two elements in the {@code interfaces} array may
      * refer to identical {@code Class} objects.

test/jdk/java/lang/reflect/Proxy/HiddenProxyInterface.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8250219
+ * @run testng HiddenProxyInterface
+ */
+
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.testng.annotations.Test;
+import static org.testng.Assert.*;
+
+public class HiddenProxyInterface {
+    private static final Path CLASSES_DIR = Paths.get(System.getProperty("test.classes"));
+
+    interface Intf {
+        void m();
+    }
+
+    @Test(expectedExceptions = { IllegalArgumentException.class })
+    public void testHiddenInterface() throws Exception {
+        Path classFile = CLASSES_DIR.resolve("HiddenProxyInterface$Intf.class");
+        byte[] bytes = Files.readAllBytes(classFile);
+        Class<?> hiddenIntf = MethodHandles.lookup().defineHiddenClass(bytes, false).lookupClass();
+        Proxy.newProxyInstance(HiddenProxyInterface.class.getClassLoader(),
+                new Class<?>[]{ hiddenIntf },
+                new InvocationHandler() {
+                    @Override
+                    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+                        return null;
+                    }
+                });
+    }
+}