8178870: instrumentation.retransformClasses cause coredump

Don't double-free cached class bytes on redefinition loading failure.

Reviewed-by: sspitsyn, jiangli

make/test/JtregNativeHotspot.gmk

@@ -59,6 +59,7 @@ BUILD_HOTSPOT_JTREG_NATIVE_SRC += \
     $(TOPDIR)/test/hotspot/jtreg/runtime/SameObject \
     $(TOPDIR)/test/hotspot/jtreg/runtime/BoolReturn \
     $(TOPDIR)/test/hotspot/jtreg/runtime/noClassDefFoundMsg \
+    $(TOPDIR)/test/hotspot/jtreg/runtime/RedefineTests \
     $(TOPDIR)/test/hotspot/jtreg/compiler/floatingpoint/ \
     $(TOPDIR)/test/hotspot/jtreg/compiler/calls \
     $(TOPDIR)/test/hotspot/jtreg/serviceability/jvmti/GetOwnedMonitorInfo \
@@ -103,6 +104,7 @@ ifeq ($(TOOLCHAIN_TYPE), solstudio)
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libMAAClassLoadPrepare := -lc
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libMAAThreadStart := -lc
     BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libAllowedFunctions := -lc
+    BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libRedefineDoubleDelete := -lc
 endif
 
 ifeq ($(OPENJDK_TARGET_OS), linux)

src/hotspot/share/prims/jvmtiRedefineClasses.cpp

@@ -158,6 +158,11 @@ bool VM_RedefineClasses::doit_prologue() {
         ClassLoaderData* cld = _scratch_classes[i]->class_loader_data();
         // Free the memory for this class at class unloading time.  Not before
         // because CMS might think this is still live.
+        InstanceKlass* ik = get_ik(_class_defs[i].klass);
+        if (ik->get_cached_class_file() == _scratch_classes[i]->get_cached_class_file()) {
+          // Don't double-free cached_class_file copied from the original class if error.
+          _scratch_classes[i]->set_cached_class_file(NULL);
+        }
         cld->add_to_deallocate_list(InstanceKlass::cast(_scratch_classes[i]));
       }
     }
@@ -3946,12 +3951,12 @@ void VM_RedefineClasses::redefine_single_class(jclass the_jclass,
   // with them was cached on the scratch class, move to the_class.
   // Note: we still want to do this if nothing needed caching since it
   // should get cleared in the_class too.
-  if (the_class->get_cached_class_file_bytes() == 0) {
+  if (the_class->get_cached_class_file() == 0) {
     // the_class doesn't have a cache yet so copy it
     the_class->set_cached_class_file(scratch_class->get_cached_class_file());
   }
-  else if (scratch_class->get_cached_class_file_bytes() !=
-           the_class->get_cached_class_file_bytes()) {
+  else if (scratch_class->get_cached_class_file() !=
+           the_class->get_cached_class_file()) {
     // The same class can be present twice in the scratch classes list or there
     // are multiple concurrent RetransformClasses calls on different threads.
     // In such cases we have to deallocate scratch_class cached_class_file.

test/hotspot/jtreg/runtime/RedefineTests/RedefineDoubleDelete.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8178870
+ * @summary Redefine class with CFLH twice to test deleting the cached_class_file
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ * @modules java.compiler
+ *          java.instrument
+ *          jdk.jartool/sun.tools.jar
+ * @run main RedefineClassHelper
+ * @run main/othervm/native -Xlog:redefine+class+load+exceptions -agentlib:RedefineDoubleDelete -javaagent:redefineagent.jar RedefineDoubleDelete
+ */
+
+public class RedefineDoubleDelete {
+
+    // Class gets a redefinition error because it adds a data member
+    public static String newB =
+                "class RedefineDoubleDelete$B {" +
+                "   int count1 = 0;" +
+                "}";
+
+    public static String newerB =
+                "class RedefineDoubleDelete$B { " +
+                "   int faa() { System.out.println(\"baa\"); return 2; }" +
+                "}";
+
+    // The ClassFileLoadHook for this class turns foo into faa and prints out faa.
+    static class B {
+      int faa() { System.out.println("foo"); return 1; }
+    }
+
+    public static void main(String args[]) throws Exception {
+
+        B b = new B();
+        int val = b.faa();
+        if (val != 1) {
+            throw new RuntimeException("return value wrong " + val);
+        }
+
+        // Redefine B twice to get cached_class_file in both B scratch classes
+        try {
+            RedefineClassHelper.redefineClass(B.class, newB);
+        } catch (java.lang.UnsupportedOperationException e) {
+            // this is expected
+        }
+        try {
+            RedefineClassHelper.redefineClass(B.class, newB);
+        } catch (java.lang.UnsupportedOperationException e) {
+            // this is expected
+        }
+
+        // Do a full GC.
+        System.gc();
+
+        // Redefine with a compatible class
+        RedefineClassHelper.redefineClass(B.class, newerB);
+        val = b.faa();
+        if (val != 2) {
+            throw new RuntimeException("return value wrong " + val);
+        }
+
+        // Do another full GC to clean things up.
+        System.gc();
+    }
+}

test/hotspot/jtreg/runtime/RedefineTests/libRedefineDoubleDelete.c

@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include "jvmti.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef JNI_ENV_ARG
+
+#ifdef __cplusplus
+#define JNI_ENV_ARG(x, y) y
+#define JNI_ENV_PTR(x) x
+#else
+#define JNI_ENV_ARG(x,y) x, y
+#define JNI_ENV_PTR(x) (*x)
+#endif
+
+#endif
+
+#define TranslateError(err) "JVMTI error"
+
+static jvmtiEnv *jvmti = NULL;
+
+static jint Agent_Initialize(JavaVM *jvm, char *options, void *reserved);
+
+JNIEXPORT
+jint JNICALL Agent_OnLoad(JavaVM *jvm, char *options, void *reserved) {
+    return Agent_Initialize(jvm, options, reserved);
+}
+
+JNIEXPORT
+jint JNICALL Agent_OnAttach(JavaVM *jvm, char *options, void *reserved) {
+    return Agent_Initialize(jvm, options, reserved);
+}
+
+JNIEXPORT
+jint JNICALL JNI_OnLoad(JavaVM *jvm, void *reserved) {
+    return JNI_VERSION_9;
+}
+
+
+static jint newClassDataLen = 0;
+static unsigned char* newClassData = NULL;
+
+static jint
+getBytecodes(jvmtiEnv *jvmti_env,
+             jint class_data_len, const unsigned char* class_data) {
+    int i;
+    jint res;
+
+    newClassDataLen = class_data_len;
+    res = (*jvmti_env)->Allocate(jvmti_env, newClassDataLen, &newClassData);
+    if (res != JNI_OK) {
+        printf("    Unable to allocate bytes\n");
+        return JNI_ERR;
+    }
+    for (i = 0; i < newClassDataLen; i++) {
+        newClassData[i] = class_data[i];
+        // Rewrite oo in class to aa
+        if (i > 0 && class_data[i] == 'o' && class_data[i-1] == 'o') {
+            newClassData[i] = newClassData[i-1] = 'a';
+        }
+    }
+    printf("  ... copied bytecode: %d bytes\n", (int)newClassDataLen);
+    return JNI_OK;
+}
+
+
+static void JNICALL
+Callback_ClassFileLoadHook(jvmtiEnv *jvmti_env, JNIEnv *env,
+                           jclass class_being_redefined,
+                           jobject loader, const char* name, jobject protection_domain,
+                           jint class_data_len, const unsigned char* class_data,
+                           jint *new_class_data_len, unsigned char** new_class_data) {
+    if (name != NULL && strcmp(name, "RedefineDoubleDelete$B") == 0) {
+        if (newClassData == NULL) {
+            jint res = getBytecodes(jvmti_env, class_data_len, class_data);
+            if (res == JNI_ERR) {
+              printf(">>>    ClassFileLoadHook event: class name %s FAILED\n", name);
+              return;
+            }
+            // Only change for first CFLH event.
+            *new_class_data_len = newClassDataLen;
+            *new_class_data = newClassData;
+        }
+        printf(">>>    ClassFileLoadHook event: class name %s\n", name);
+    }
+}
+
+static
+jint Agent_Initialize(JavaVM *jvm, char *options, void *reserved) {
+    jint res, size;
+    jvmtiCapabilities caps;
+    jvmtiEventCallbacks callbacks;
+    jvmtiError err;
+
+    res = JNI_ENV_PTR(jvm)->GetEnv(JNI_ENV_ARG(jvm, (void **) &jvmti),
+        JVMTI_VERSION_9);
+    if (res != JNI_OK || jvmti == NULL) {
+        printf("    Error: wrong result of a valid call to GetEnv!\n");
+        return JNI_ERR;
+    }
+
+    printf("Enabling following capabilities: can_generate_all_class_hook_events, "
+           "can_retransform_classes, can_redefine_classes");
+    memset(&caps, 0, sizeof(caps));
+    caps.can_generate_all_class_hook_events = 1;
+    caps.can_retransform_classes = 1;
+    caps.can_redefine_classes = 1;
+    printf("\n");
+
+    err = (*jvmti)->AddCapabilities(jvmti, &caps);
+    if (err != JVMTI_ERROR_NONE) {
+        printf("    Error in AddCapabilites: %s (%d)\n", TranslateError(err), err);
+        return JNI_ERR;
+    }
+
+    size = (jint)sizeof(callbacks);
+
+    memset(&callbacks, 0, sizeof(callbacks));
+    callbacks.ClassFileLoadHook = Callback_ClassFileLoadHook;
+
+    err = (*jvmti)->SetEventCallbacks(jvmti, &callbacks, size);
+    if (err != JVMTI_ERROR_NONE) {
+        printf("    Error in SetEventCallbacks: %s (%d)\n", TranslateError(err), err);
+        return JNI_ERR;
+    }
+
+    err = (*jvmti)->SetEventNotificationMode(jvmti, JVMTI_ENABLE, JVMTI_EVENT_CLASS_FILE_LOAD_HOOK, NULL);
+    if (err != JVMTI_ERROR_NONE) {
+        printf("    Error in SetEventNotificationMode: %s (%d)\n", TranslateError(err), err);
+        return JNI_ERR;
+    }
+
+    return JNI_OK;
+}
+
+#ifdef __cplusplus
+}
+#endif