8198393: Instrumentation.retransformClasses() throws NullPointerException when handling a zero-length array

Reviewed-by: sspitsyn, cjplummer

src/java.instrument/share/classes/sun/instrument/InstrumentationImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -161,6 +161,9 @@ public class InstrumentationImpl implements Instrumentation {
             throw new UnsupportedOperationException(
               "retransformClasses is not supported in this environment");
         }
+        if (classes.length == 0) {
+            return; // no-op
+        }
         retransformClasses0(mNativeAgent, classes);
     }
 

test/hotspot/jtreg/serviceability/jvmti/RetransformClassesZeroLength.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8198393
+ * @summary Instrumentation.retransformClasses(new Class[0]) should be NOP
+ * @library /test/lib
+ * @modules java.instrument
+ * @compile RetransformClassesZeroLength.java
+ * @run main RetransformClassesZeroLength
+ */
+
+import java.lang.instrument.ClassFileTransformer;
+import java.lang.instrument.IllegalClassFormatException;
+import java.lang.instrument.Instrumentation;
+import java.lang.instrument.UnmodifiableClassException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.ProtectionDomain;
+
+import jdk.test.lib.process.ProcessTools;
+
+
+public class RetransformClassesZeroLength {
+
+    private static String manifest =
+            "Premain-Class: " + RetransformClassesZeroLength.Agent.class.getName() + "\n"
+            + "Can-Retransform-Classes: true\n";
+
+    private static String CP = System.getProperty("test.classes");
+
+    public static void main(String args[]) throws Throwable {
+        String agentJar = buildAgent();
+        ProcessTools.executeProcess(
+                ProcessTools.createJavaProcessBuilder(
+                        "-javaagent:" + agentJar,
+                        "-version")
+        ).shouldHaveExitValue(0);
+    }
+
+    private static String buildAgent() throws Exception {
+        Path jar = Files.createTempFile(Paths.get("."), null, ".jar");
+        String jarPath = jar.toAbsolutePath().toString();
+        ClassFileInstaller.writeJar(jarPath,
+                ClassFileInstaller.Manifest.fromString(manifest),
+                RetransformClassesZeroLength.class.getName());
+        return jarPath;
+    }
+
+
+    public static class Agent implements ClassFileTransformer {
+        public static void premain(String args, Instrumentation inst) {
+            inst.addTransformer(new NoOpTransformer());
+            try {
+                inst.retransformClasses(new Class[0]);
+            } catch (UnmodifiableClassException ex) {
+                throw new AssertionError(ex);
+            }
+        }
+    }
+
+    private static class NoOpTransformer implements ClassFileTransformer {
+        @Override
+        public byte[] transform(ClassLoader loader,
+                                String className,
+                                Class<?> classBeingRedefined,
+                                ProtectionDomain protectionDomain,
+                                byte[] classfileBuffer
+                                ) throws IllegalClassFormatException {
+            return null;    // no transform
+        }
+    }
+}