8202669: Intermittent crash in ClassLoadingService::compute_class_size()

Don't report classes that failed to load when reporting class unloading. Remove the classes first.

Reviewed-by: dholmes, lfoltan, ccheung

src/hotspot/share/classfile/classLoaderData.cpp

@@ -573,9 +573,6 @@ void ClassLoaderData::remove_class(Klass* scratch_class) {
 void ClassLoaderData::unload() {
   _unloading = true;
 
-  // Tell serviceability tools these classes are unloading
-  classes_do(InstanceKlass::notify_unload_class);
-
   LogTarget(Debug, class, loader, data) lt;
   if (lt.is_enabled()) {
     ResourceMark rm;
@@ -589,6 +586,10 @@ void ClassLoaderData::unload() {
   // if they are not already on the _klasses list.
   unload_deallocate_list();
 
+  // Tell serviceability tools these classes are unloading
+  // after erroneous classes are released.
+  classes_do(InstanceKlass::notify_unload_class);
+
   // Clean up global class iterator for compiler
   static_klass_iterator.adjust_saved_class(this);
 }

src/hotspot/share/services/classLoadingService.cpp

@@ -172,7 +172,9 @@ size_t ClassLoadingService::compute_class_size(InstanceKlass* k) {
     // FIXME: Need to count the contents of methods
     class_size += k->constants()->size();
     class_size += k->local_interfaces()->size();
-    class_size += k->transitive_interfaces()->size();
+    if (k->transitive_interfaces() != NULL) {
+      class_size += k->transitive_interfaces()->size();
+    }
     // We do not have to count implementors, since we only store one!
     // FIXME: How should these be accounted for, now when they have moved.
     //class_size += k->fields()->size();

test/hotspot/jtreg/runtime/BadObjectClass/TestUnloadClassError.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+/*
+ * @test
+ * @bug 6194361
+ * @bug 8202669
+ * @summary Make sure the VM doesn't crash and throws a SecurityException
+ *          if defineClass() is called on a byte buffer that parses into a invalid
+ *          java.lang.Object class.
+ *          Also, make sure the vm doesn't crash on notification for unloading an invalid
+ *          java.lang.Object class.
+ * @library /runtime/testlibrary /test/lib
+ * @modules java.base/jdk.internal.misc
+ *          java.compiler
+ * @run main TestUnloadClassError
+ */
+
+import jdk.test.lib.compiler.InMemoryJavaCompiler;
+
+public class TestUnloadClassError extends ClassLoader {
+
+   static String source =
+       " package java.lang;" +
+       " public class Object" +
+       " {" +
+       "   int field;" +
+       "   public boolean equals(Object o) {" +
+       "     System.out.println(o.field);" +
+       "     return false;" +
+       "   }" +
+       " }";
+
+  public static void main(String[] args) throws Exception
+  {
+    try {
+      TestUnloadClassError loader = new TestUnloadClassError();
+      byte[] buf = InMemoryJavaCompiler.compile("java.lang.Object", source,
+                                                "--patch-module=java.base");
+      Class c = loader.defineClass(buf, 0, buf.length);
+      System.out.println("test FAILS");
+      throw new RuntimeException("Did not get security exception");
+    } catch(SecurityException e) {
+      System.out.println("test expects SecurityException");
+    }
+
+    // Unload bad class
+    ClassUnloadCommon.triggerUnloading();
+    System.out.println("test PASSES if it doesn't crash");
+  }
+}