8076264: [macosx] Launching app on MacOSX requires enclosing class

Reviewed-by: azvegint, ksrini

jdk/src/java.base/macosx/native/libjli/java_md_macosx.c

@@ -948,8 +948,15 @@ SetMainClassForAWT(JNIEnv *env, jclass mainClass) {
     jmethodID getCanonicalNameMID = NULL;
     NULL_CHECK(getCanonicalNameMID = (*env)->GetMethodID(env, classClass, "getCanonicalName", "()Ljava/lang/String;"));
 
-    jstring mainClassString = NULL;
-    NULL_CHECK(mainClassString = (*env)->CallObjectMethod(env, mainClass, getCanonicalNameMID));
+    jstring mainClassString = (*env)->CallObjectMethod(env, mainClass, getCanonicalNameMID);
+    if ((*env)->ExceptionCheck(env)) {
+        /*
+         * Clears all errors caused by getCanonicalName() on the mainclass and
+         * leaves the JAVA_MAIN_CLASS__<pid> empty.
+         */
+        (*env)->ExceptionClear(env);
+        return;
+    }
 
     const char *mainClassName = NULL;
     NULL_CHECK(mainClassName = (*env)->GetStringUTFChars(env, mainClassString, NULL));
@@ -1056,7 +1063,7 @@ JVMInit(InvocationFunctions* ifn, jlong threadStackSize,
  * Note the jvmInstance must be initialized first before entering into
  * ShowSplashScreen, as there is a callback into the JLI_GetJavaVMInstance.
  */
-void PostJVMInit(JNIEnv *env, jstring mainClass, JavaVM *vm) {
+void PostJVMInit(JNIEnv *env, jclass mainClass, JavaVM *vm) {
     jvmInstance = vm;
     SetMainClassForAWT(env, mainClass);
     CHECK_EXCEPTION_RETURN();

jdk/src/java.base/share/native/libjli/java.h

@@ -194,7 +194,7 @@ void InitLauncher(jboolean javaw);
  * For MacOSX and Windows/Unix compatibility we require these
  * entry points, some of them may be stubbed out on Windows/Unixes.
  */
-void     PostJVMInit(JNIEnv *env, jstring mainClass, JavaVM *vm);
+void     PostJVMInit(JNIEnv *env, jclass mainClass, JavaVM *vm);
 void     ShowSplashScreen();
 void     RegisterThread();
 /*

jdk/src/java.base/unix/native/libjli/java_md_solinux.c

@@ -938,7 +938,7 @@ JVMInit(InvocationFunctions* ifn, jlong threadStackSize,
 }
 
 void
-PostJVMInit(JNIEnv *env, jstring mainClass, JavaVM *vm)
+PostJVMInit(JNIEnv *env, jclass mainClass, JavaVM *vm)
 {
     // stubbed out for windows and *nixes.
 }

jdk/src/java.base/windows/native/libjli/java_md.c

@@ -970,7 +970,7 @@ JVMInit(InvocationFunctions* ifn, jlong threadStackSize,
 }
 
 void
-PostJVMInit(JNIEnv *env, jstring mainClass, JavaVM *vm)
+PostJVMInit(JNIEnv *env, jclass mainClass, JavaVM *vm)
 {
     // stubbed out for windows and *nixes.
 }

jdk/test/tools/launcher/TestMainWithoutEnclosing.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @test
+ * @bug 8076264
+ * @summary Launching app shouldn't require enclosing class for the main class.
+ * @compile TestMainWithoutEnclosing.java
+ * @run main TestMainWithoutEnclosing
+ */
+public final class TestMainWithoutEnclosing extends TestHelper {
+
+    static final String EnclosingName = "Enclosing";
+
+    static void createJarFile(File testJar) throws IOException {
+        List<String> scratch = new ArrayList<>();
+        scratch.add("public class Enclosing {");
+        scratch.add("    public static final class Main {");
+        scratch.add("        public static void main(String... args) {");
+        scratch.add("            System.out.println(\"Hello World\");");
+        scratch.add("        }");
+        scratch.add("    }");
+        scratch.add("}");
+        File enclosingFile = new File(EnclosingName + ".java");
+        createFile(enclosingFile, scratch);
+        compile(enclosingFile.getName());
+        // avoid side effects remove the Enclosing class
+        getClassFile(enclosingFile).delete();
+        createJar("cvfe", testJar.getName(), EnclosingName + "$Main",
+                EnclosingName + "$Main" + ".class");
+        // remove extraneous files in the current directory
+        new File(EnclosingName + "$Main" + ".class").delete();
+    }
+
+    public static void main(String... args) throws IOException {
+        File testJarFile = new File("test.jar");
+        createJarFile(testJarFile);
+        TestResult tr = doExec(javaCmd, "-jar", testJarFile.getName());
+        if (!tr.isOK()) {
+            System.out.println(tr);
+            throw new RuntimeException("test returned non-positive value");
+        }
+        if (!tr.contains("Hello World")) {
+            System.out.println(tr);
+            throw new RuntimeException("expected output not found");
+        }
+    }
+}