8160950: Agent JAR added to app class loader rather than system class loader when running with -Djava.system.class.loader

Add agent JAR to the custom system class loader

Reviewed-by: alanb, mchung, dholmes

jdk/src/java.instrument/share/classes/java/lang/instrument/package.html

@@ -105,6 +105,10 @@ method is not invoked.
 The agent class will be loaded by the system class loader
 (see {@link java.lang.ClassLoader#getSystemClassLoader ClassLoader.getSystemClassLoader}). This is
 the class loader which typically loads the class containing the application <code>main</code> method.
+The system class loader must support a mechanism to add an agent JAR file to the system class path.
+If it is a custom system class loader then it must define the
+<code>appendToClassPathForInstrumentation</code> method as specified in
+{@link Instrumentation#appendToSystemClassLoaderSearch appendToSystemClassLoaderSearch}.
 The <code>premain</code> methods will be run under the same security and classloader 
 rules as the application <code>main</code> method.
 There are no modeling restrictions on what the agent <code>premain</code> method may do.
@@ -140,7 +144,10 @@ supports the starting of agents after the VM has started the following applies:
 
   <li><p>The system class loader (
       {@link java.lang.ClassLoader#getSystemClassLoader ClassLoader.getSystemClassLoader}) must
-      support a mechanism to add an agent JAR file to the system class path.</li>
+      support a mechanism to add an agent JAR file to the system class path.
+      If it is a custom system class loader then it must define the
+      <code>appendToClassPathForInstrumentation</code> method as specified in
+      {@link Instrumentation#appendToSystemClassLoaderSearch appendToSystemClassLoaderSearch}.</li>
 </ol>  
 
 <P>

jdk/src/java.instrument/share/native/libinstrument/InvocationAdapter.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -190,10 +190,8 @@ DEF_Agent_OnLoad(JavaVM *vm, char *tail, void * reserved) {
             return JNI_ERR;
         }
 
-        /*
+        /* Save the jarfile name */
-         * Add to the jarfile
+        agent->mJarfile = jarfile;
-         */
-        appendClassPath(agent, jarfile);
 
         /*
          * The value of the Premain-Class attribute becomes the agent
@@ -241,7 +239,6 @@ DEF_Agent_OnLoad(JavaVM *vm, char *tail, void * reserved) {
         /*
          * Clean-up
          */
-        free(jarfile);
         if (options != NULL) free(options);
         freeAttributes(attributes);
         free(premainClass);
@@ -459,7 +456,23 @@ eventHandlerVMInit( jvmtiEnv *      jvmtienv,
 
     /* process the premain calls on the all the JPL agents */
     if ( environment != NULL ) {
-        jthrowable outstandingException = preserveThrowable(jnienv);
+        jthrowable outstandingException = NULL;
+        /*
+         * Add the jarfile to the system class path
+         */
+        JPLISAgent * agent = environment->mAgent;
+        if (appendClassPath(agent, agent->mJarfile)) {
+            fprintf(stderr, "Unable to add %s to system class path - "
+                    "the system class loader does not define the "
+                    "appendToClassPathForInstrumentation method or the method failed\n",
+                    agent->mJarfile);
+            free((void *)agent->mJarfile);
+            abortJVM(jnienv, JPLIS_ERRORMESSAGE_CANNOTSTART);
+        }
+        free((void *)agent->mJarfile);
+        agent->mJarfile = NULL;
+
+        outstandingException = preserveThrowable(jnienv);
         success = processJavaStart( environment->mAgent,
                                     jnienv);
         restoreThrowable(jnienv, outstandingException);
@@ -631,32 +644,19 @@ appendClassPath( JPLISAgent* agent,
     jvmtierr = (*jvmtienv)->AddToSystemClassLoaderSearch(jvmtienv, jarfile);
     check_phase_ret_1(jvmtierr);
 
-    if (jvmtierr == JVMTI_ERROR_NONE) {
+    switch (jvmtierr) {
-        return 0;
+        case JVMTI_ERROR_NONE :
-    } else {
+            return 0;
-        jvmtiPhase phase;
+        case JVMTI_ERROR_CLASS_LOADER_UNSUPPORTED :
-        jvmtiError err;
+            fprintf(stderr, "System class loader does not define "
-
+                "the appendToClassPathForInstrumentation method\n");
-        err = (*jvmtienv)->GetPhase(jvmtienv, &phase);
+            break;
-        /* can be called from any phase */
+        default:
-        jplis_assert(err == JVMTI_ERROR_NONE);
+            fprintf(stderr, "Unexpected error (%d) returned by "
-
+                "AddToSystemClassLoaderSearch\n", jvmtierr);
-        if (phase == JVMTI_PHASE_LIVE) {
+            break;
-            switch (jvmtierr) {
-                case JVMTI_ERROR_CLASS_LOADER_UNSUPPORTED :
-                    fprintf(stderr, "System class loader does not support adding "
-                        "JAR file to system class path during the live phase!\n");
-                        break;
-                default:
-                    fprintf(stderr, "Unexpected error (%d) returned by "
-                        "AddToSystemClassLoaderSearch\n", jvmtierr);
-                    break;
-            }
-            return -1;
-        }
-        jplis_assert(0);
     }
-    return -2;
+    return -1;
 }
 
 

jdk/src/java.instrument/share/native/libinstrument/JPLISAgent.c

@@ -272,6 +272,7 @@ initializeJPLISAgent(   JPLISAgent *    agent,
     agent->mNativeMethodPrefixAdded                  = JNI_FALSE;
     agent->mAgentClassName                           = NULL;
     agent->mOptionsString                            = NULL;
+    agent->mJarfile                                  = NULL;
 
     /* make sure we can recover either handle in either direction.
      * the agent has a ref to the jvmti; make it mutual

jdk/src/java.instrument/share/native/libinstrument/JPLISAgent.h

@@ -107,6 +107,7 @@ struct _JPLISAgent {
     jboolean                mNativeMethodPrefixAdded;     /* indicates if can_set_native_method_prefix capability has been added */
     char const *            mAgentClassName;        /* agent class name */
     char const *            mOptionsString;         /* -javaagent options string */
+    const char *            mJarfile;               /* agent jar file name */
 };
 
 /*

jdk/test/java/lang/instrument/CustomSystemLoader/Agent.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.PrintStream;
+import java.lang.instrument.*;
+import java.lang.reflect.Field;
+
+/**
+  * @test
+  * @bug 8160950
+  * @summary test for custom system class loader
+  *
+  * @run build App Agent CustomLoader
+  * @run shell ../MakeJAR3.sh Agent 'Can-Retransform-Classes: true'
+  * @run main/othervm -javaagent:Agent.jar -Djava.system.class.loader=CustomLoader App
+  */
+
+public class Agent {
+    private static PrintStream err = System.err;
+    private static PrintStream out = System.out;
+    public  static boolean failed = false;
+
+    public static void premain(String agentArgs, Instrumentation instrumentation) {
+        ClassLoader myClassLoader = Agent.class.getClassLoader();
+        out.println("Agent: started; myClassLoader: " + myClassLoader);
+        try {
+            Field fld = myClassLoader.getClass().getField("agentClassLoader");
+            fld.set(myClassLoader.getClass(), myClassLoader);
+        } catch (Exception ex) {
+            failed = true;
+            ex.printStackTrace();
+        }
+        out.println("Agent: finished");
+   }
+}

jdk/test/java/lang/instrument/CustomSystemLoader/App.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.PrintStream;
+
+public class App {
+
+    public static void main(String args[]) throws Exception {
+        (new App()).run(args, System.out);
+    }
+
+    public void run(String args[], PrintStream out) throws Exception {
+        out.println("App: Test started");
+        if (CustomLoader.agentClassLoader != CustomLoader.myself) {
+            System.out.println("App: agentClassLoader:    " + CustomLoader.agentClassLoader);
+            System.out.println("App: CustomLoader.myself: " + CustomLoader.myself);
+            System.out.println("App: myClassLoader:       " + App.class.getClassLoader());
+            throw new Exception("App: Agent's system class loader is not CustomLoader");
+        } else if (Agent.failed) {
+            throw new Exception("App: Agent failed");
+        } else if (CustomLoader.failed) {
+            throw new Exception("App: CustomLoader failed");
+        }
+        out.println("App: Test passed");
+    }
+}

jdk/test/java/lang/instrument/CustomSystemLoader/CustomLoader.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.DataInputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.jar.JarFile;
+import java.util.zip.ZipEntry;
+
+public class CustomLoader extends ClassLoader {
+    private static PrintStream out = System.out;
+    public  static ClassLoader myself;
+    public  static ClassLoader agentClassLoader;
+    public  static boolean failed = true;
+
+    public CustomLoader(ClassLoader classLoader) {
+        super(classLoader);
+        myself = this;
+    }
+
+    @Override
+    public Class<?> loadClass(String name) throws ClassNotFoundException {
+        out.println("CustomLoader: loading class: " + name);
+        if (name.equals("Agent")) {
+            Class c = null;
+            try {
+                byte[] buf = locateBytes();
+                c = defineClass(name, buf, 0, buf.length);
+            } catch (IOException ex) {
+                throw new ClassNotFoundException(ex.getMessage());
+            }
+            resolveClass(c);
+            out.println("CustomLoader.loadClass after resolveClass: " + name +
+                        "; Class: " + c + "; ClassLoader: " + c.getClassLoader());
+            return c;
+        }
+        return super.loadClass(name);
+    }
+
+    private byte[] locateBytes() throws IOException {
+        try {
+            JarFile jar = new JarFile("Agent.jar");
+            InputStream is = jar.getInputStream(jar.getEntry("Agent.class"));
+            int len = is.available();
+            byte[] buf = new byte[len];
+            DataInputStream in = new DataInputStream(is);
+            in.readFully(buf);
+            return buf;
+        } catch (IOException ioe) {
+            ioe.printStackTrace();
+            throw new IOException("Test failed due to IOException!");
+        }
+    }
+
+    void appendToClassPathForInstrumentation(String path) {
+        out.println("CustomLoader.appendToClassPathForInstrumentation: " +
+                    this + ", jar: " + path);
+        failed = false;
+    }
+}