8338526: Don't store abstract and interface Klasses in class metaspace

Reviewed-by: stuefe, iklam

src/hotspot/share/classfile/classFileParser.hpp

@@ -547,6 +547,7 @@ class ClassFileParser {
 
   bool is_hidden() const { return _is_hidden; }
   bool is_interface() const { return _access_flags.is_interface(); }
+  bool is_abstract() const { return _access_flags.is_abstract(); }
 
   ClassLoaderData* loader_data() const { return _loader_data; }
   const Symbol* class_name() const { return _class_name; }

src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,6 @@
 #include "jfr/support/jfrThreadLocal.hpp"
 #include "jfr/utilities/jfrEpochQueue.inline.hpp"
 #include "jfr/utilities/jfrTypes.hpp"
-#include "memory/metaspace.hpp"
 #include "oops/compressedKlass.inline.hpp"
 #include "utilities/macros.hpp"
 
@@ -75,13 +74,14 @@ static size_t element_size(bool compressed) {
   return compressed ? NARROW_ELEMENT_SIZE : ELEMENT_SIZE;
 }
 
-static bool can_compress_element(traceid id) {
+static bool can_compress_element(const Klass* klass) {
-  return Metaspace::using_class_space() && id < uncompressed_threshold;
+  return CompressedKlassPointers::is_in_encoding_range(klass) &&
+         JfrTraceId::load_raw(klass) < uncompressed_threshold;
 }
 
 static size_t element_size(const Klass* klass) {
   assert(klass != nullptr, "invariant");
-  return element_size(can_compress_element(JfrTraceId::load_raw(klass)));
+  return element_size(can_compress_element(klass));
 }
 
 static bool is_unloaded(traceid id, bool previous_epoch) {
@@ -137,7 +137,8 @@ static inline void store_traceid(JfrEpochQueueNarrowKlassElement* element, trace
 }
 
 static void store_compressed_element(traceid id, const Klass* klass, u1* pos) {
-  assert(can_compress_element(id), "invariant");
+  assert(can_compress_element(klass), "invariant");
+  assert(id == JfrTraceId::load_raw(klass), "invariant");
   JfrEpochQueueNarrowKlassElement* const element = new (pos) JfrEpochQueueNarrowKlassElement();
   store_traceid(element, id);
   element->compressed_klass = encode(klass);
@@ -153,7 +154,7 @@ static void store_element(const Klass* klass, u1* pos) {
   assert(pos != nullptr, "invariant");
   assert(klass != nullptr, "invariant");
   const traceid id = JfrTraceId::load_raw(klass);
-  if (can_compress_element(id)) {
+  if (can_compress_element(klass)) {
     store_compressed_element(id, klass, pos);
     return;
   }

src/hotspot/share/memory/allocation.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -75,14 +75,16 @@ void* MetaspaceObj::operator new(size_t size, ClassLoaderData* loader_data,
                                  size_t word_size,
                                  MetaspaceObj::Type type, TRAPS) throw() {
   // Klass has its own operator new
-  return Metaspace::allocate(loader_data, word_size, type, THREAD);
+  assert(type != ClassType, "class has its own operator new");
+  return Metaspace::allocate(loader_data, word_size, type, /*use_class_space*/ false, THREAD);
 }
 
 void* MetaspaceObj::operator new(size_t size, ClassLoaderData* loader_data,
                                  size_t word_size,
                                  MetaspaceObj::Type type) throw() {
   assert(!Thread::current()->is_Java_thread(), "only allowed by non-Java thread");
-  return Metaspace::allocate(loader_data, word_size, type);
+  assert(type != ClassType, "class has its own operator new");
+  return Metaspace::allocate(loader_data, word_size, type, /*use_class_space*/ false);
 }
 
 bool MetaspaceObj::is_valid(const MetaspaceObj* p) {

src/hotspot/share/memory/metaspace.cpp

@@ -830,7 +830,7 @@ size_t Metaspace::max_allocation_word_size() {
 // is suitable for calling from non-Java threads.
 // Callers are responsible for checking null.
 MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
-                              MetaspaceObj::Type type) {
+                              MetaspaceObj::Type type, bool use_class_space) {
   assert(word_size <= Metaspace::max_allocation_word_size(),
          "allocation size too large (" SIZE_FORMAT ")", word_size);
 
@@ -840,7 +840,7 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
   // Deal with concurrent unloading failed allocation starvation
   MetaspaceCriticalAllocation::block_if_concurrent_purge();
 
-  MetadataType mdtype = (type == MetaspaceObj::ClassType) ? ClassType : NonClassType;
+  MetadataType mdtype = use_class_space ? ClassType : NonClassType;
 
   // Try to allocate metadata.
   MetaWord* result = loader_data->metaspace_non_null()->allocate(word_size, mdtype);
@@ -856,7 +856,7 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
 }
 
 MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
-                              MetaspaceObj::Type type, TRAPS) {
+                              MetaspaceObj::Type type, bool use_class_space, TRAPS) {
 
   if (HAS_PENDING_EXCEPTION) {
     assert(false, "Should not allocate with exception pending");
@@ -864,10 +864,10 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size,
   }
   assert(!THREAD->owns_locks(), "allocating metaspace while holding mutex");
 
-  MetaWord* result = allocate(loader_data, word_size, type);
+  MetaWord* result = allocate(loader_data, word_size, type, use_class_space);
 
   if (result == nullptr) {
-    MetadataType mdtype = (type == MetaspaceObj::ClassType) ? ClassType : NonClassType;
+    MetadataType mdtype = use_class_space ? ClassType : NonClassType;
     tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype);
 
     // Allocation failed.

src/hotspot/share/memory/metaspace.hpp

@@ -110,12 +110,12 @@ public:
   static size_t max_allocation_word_size();
 
   static MetaWord* allocate(ClassLoaderData* loader_data, size_t word_size,
-                            MetaspaceObj::Type type, TRAPS);
+                            MetaspaceObj::Type type, bool use_class_space, TRAPS);
 
   // Non-TRAPS version of allocate which can be called by a non-Java thread, that returns
   // null on failure.
   static MetaWord* allocate(ClassLoaderData* loader_data, size_t word_size,
-                            MetaspaceObj::Type type);
+                            MetaspaceObj::Type type, bool use_class_space);
 
   // Returns true if the pointer points into class space, non-class metaspace, or the
   // metadata portion of the CDS archive.

src/hotspot/share/oops/annotations.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -89,7 +89,6 @@ class Annotations: public MetaspaceObj {
   // Turn metadata annotations into a Java heap object (oop)
   static typeArrayOop make_java_array(AnnotationArray* annotations, TRAPS);
 
-  bool is_klass() const { return false; }
   void metaspace_pointers_do(MetaspaceClosure* it);
   MetaspaceObj::Type type() const { return AnnotationsType; }
 

src/hotspot/share/oops/array.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,7 +34,7 @@ template <typename T>
 inline void* Array<T>::operator new(size_t size, ClassLoaderData* loader_data, int length, TRAPS) throw() {
   size_t word_size = Array::size(length);
   return (void*) Metaspace::allocate(loader_data, word_size,
-                                     MetaspaceObj::array_type(sizeof(T)), THREAD);
+                                     MetaspaceObj::array_type(sizeof(T)), false, THREAD);
 }
 
 #endif // SHARE_OOPS_ARRAY_INLINE_HPP

src/hotspot/share/oops/arrayKlass.cpp

@@ -42,6 +42,10 @@
 #include "oops/oop.inline.hpp"
 #include "runtime/handles.inline.hpp"
 
+void* ArrayKlass::operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, TRAPS) throw() {
+  return Metaspace::allocate(loader_data, word_size, MetaspaceObj::ClassType, true, THREAD);
+}
+
 ArrayKlass::ArrayKlass() {
   assert(CDSConfig::is_dumping_static_archive() || CDSConfig::is_using_archive(), "only for CDS");
 }

src/hotspot/share/oops/arrayKlass.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -49,6 +49,8 @@ class ArrayKlass: public Klass {
   ArrayKlass(Symbol* name, KlassKind kind);
   ArrayKlass();
 
+  void* operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, TRAPS) throw();
+
  public:
   // Testing operation
   DEBUG_ONLY(bool is_array_klass_slow() const { return true; })

src/hotspot/share/oops/compressedKlass.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -107,6 +107,18 @@ public:
 
   static inline narrowKlass encode_not_null(Klass* v);
   static inline narrowKlass encode(Klass* v);
+
+  // Returns whether the pointer is in the memory region used for encoding compressed
+  // class pointers.  This includes CDS.
+
+  // encoding                                                               encoding
+  // base                                                                   end (base+range)
+  // |-----------------------------------------------------------------------|
+  // |----CDS---| |--------------------class space---------------------------|
+
+  static inline bool is_in_encoding_range(const void* p) {
+    return p >= _base && p < (_base + _range);
+  }
 };
 
 #endif // SHARE_OOPS_COMPRESSEDKLASS_HPP

src/hotspot/share/oops/constMethod.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -470,7 +470,6 @@ public:
 
   // Deallocation for RedefineClasses
   void deallocate_contents(ClassLoaderData* loader_data);
-  bool is_klass() const { return false; }
   DEBUG_ONLY(bool on_stack() { return false; })
 
   void metaspace_pointers_do(MetaspaceClosure* it);

src/hotspot/share/oops/cpCache.hpp

@@ -231,7 +231,6 @@ class ConstantPoolCache: public MetaspaceObj {
   // RedefineClasses support
   DEBUG_ONLY(bool on_stack() { return false; })
   void deallocate_contents(ClassLoaderData* data);
-  bool is_klass() const { return false; }
   void record_gc_epoch();
   uint64_t gc_epoch() { return _gc_epoch; }
 

src/hotspot/share/oops/instanceKlass.cpp

@@ -437,6 +437,11 @@ const char* InstanceKlass::nest_host_error() {
   }
 }
 
+void* InstanceKlass::operator new(size_t size, ClassLoaderData* loader_data, size_t word_size,
+                                  bool use_class_space, TRAPS) throw() {
+  return Metaspace::allocate(loader_data, word_size, ClassType, use_class_space, THREAD);
+}
+
 InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& parser, TRAPS) {
   const int size = InstanceKlass::size(parser.vtable_size(),
                                        parser.itable_size(),
@@ -449,23 +454,24 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par
   assert(loader_data != nullptr, "invariant");
 
   InstanceKlass* ik;
+  const bool use_class_space = !parser.is_interface() && !parser.is_abstract();
 
   // Allocation
   if (parser.is_instance_ref_klass()) {
     // java.lang.ref.Reference
-    ik = new (loader_data, size, THREAD) InstanceRefKlass(parser);
+    ik = new (loader_data, size, use_class_space, THREAD) InstanceRefKlass(parser);
   } else if (class_name == vmSymbols::java_lang_Class()) {
     // mirror - java.lang.Class
-    ik = new (loader_data, size, THREAD) InstanceMirrorKlass(parser);
+    ik = new (loader_data, size, use_class_space, THREAD) InstanceMirrorKlass(parser);
   } else if (is_stack_chunk_class(class_name, loader_data)) {
     // stack chunk
-    ik = new (loader_data, size, THREAD) InstanceStackChunkKlass(parser);
+    ik = new (loader_data, size, use_class_space, THREAD) InstanceStackChunkKlass(parser);
   } else if (is_class_loader(class_name, parser)) {
     // class loader - java.lang.ClassLoader
-    ik = new (loader_data, size, THREAD) InstanceClassLoaderKlass(parser);
+    ik = new (loader_data, size, use_class_space, THREAD) InstanceClassLoaderKlass(parser);
   } else {
     // normal
-    ik = new (loader_data, size, THREAD) InstanceKlass(parser);
+    ik = new (loader_data, size, use_class_space, THREAD) InstanceKlass(parser);
   }
 
   // Check for pending exception before adding to the loader data and incrementing

src/hotspot/share/oops/instanceKlass.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_OOPS_INSTANCEKLASS_HPP
 #define SHARE_OOPS_INSTANCEKLASS_HPP
 
+#include "memory/allocation.hpp"
 #include "memory/referenceType.hpp"
 #include "oops/annotations.hpp"
 #include "oops/constMethod.hpp"
@@ -144,6 +145,8 @@ class InstanceKlass: public Klass {
  protected:
   InstanceKlass(const ClassFileParser& parser, KlassKind kind = Kind, ReferenceType reference_type = REF_NONE);
 
+  void* operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, bool use_class_space, TRAPS) throw();
+
  public:
   InstanceKlass();
 

src/hotspot/share/oops/klass.cpp

@@ -247,10 +247,6 @@ Method* Klass::uncached_lookup_method(const Symbol* name, const Symbol* signatur
   return nullptr;
 }
 
-void* Klass::operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, TRAPS) throw() {
-  return Metaspace::allocate(loader_data, word_size, MetaspaceObj::ClassType, THREAD);
-}
-
 Klass::Klass() : _kind(UnknownKlassKind) {
   assert(CDSConfig::is_dumping_static_archive() || CDSConfig::is_using_archive(), "only for cds");
 }

src/hotspot/share/oops/klass.hpp

@@ -206,8 +206,6 @@ protected:
   Klass(KlassKind kind);
   Klass();
 
-  void* operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, TRAPS) throw();
-
  public:
   int kind() { return _kind; }
 

src/hotspot/share/oops/recordComponent.hpp

@@ -83,8 +83,6 @@ class RecordComponent: public MetaspaceObj {
     static bool is_read_only_by_default() { return true; }
     DEBUG_ONLY(bool on_stack() { return false; })  // for template
 
-    bool is_klass() const { return false; }
-
 #ifndef PRODUCT
     void print_on(outputStream* st) const;
 #endif

src/java.base/share/classes/java/lang/invoke/InvokerBytecodeGenerator.java

@@ -246,7 +246,7 @@ class InvokerBytecodeGenerator {
             return ClassFile.of().build(classDesc, new Consumer<>() {
                 @Override
                 public void accept(ClassBuilder clb) {
-                    clb.withFlags(ACC_FINAL | ACC_SUPER)
+                    clb.withFlags(ACC_ABSTRACT | ACC_SUPER)
                        .withSuperclass(INVOKER_SUPER_DESC)
                        .with(SourceFileAttribute.of(clb.constantPool().utf8Entry(SOURCE_PREFIX + name)));
                     config.accept(clb);

test/hotspot/jtreg/vmTestbase/metaspace/shrink_grow/ShrinkGrowTest/ShrinkGrowTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,7 @@
  * @library /vmTestbase /test/lib
  * @run main/othervm
  *      -XX:MetaspaceSize=10m
- *      -XX:MaxMetaspaceSize=20m
+ *      -XX:MaxMetaspaceSize=10m
  *      -Xlog:gc*:gc.log
  *      metaspace.shrink_grow.ShrinkGrowTest.ShrinkGrowTest
  */
@@ -52,7 +52,7 @@ import java.util.Map;
 /**
  * This is the main test in the metaspace shrink/grow series.
  *
- * It tries to allocate all available metespace (loads new classes and keeps
+ * It tries to allocate all available metaspace (loads new classes and keeps
  * them in map), then checks that loading new classes causes OOM.
  * After that it does cleanup loaded classes and then expect the new classes
  * could be loaded again.
@@ -87,7 +87,7 @@ public class ShrinkGrowTest {
      * @param classesToLoad - the limit of classes to load expecting OOM
      */
     public ShrinkGrowTest(String name, int classesToLoad) {
-        whoAmI = name;
+        whoAmI = "%" + name + "%";
         maxClassesToLoad = classesToLoad;
 
     }
@@ -98,15 +98,15 @@ public class ShrinkGrowTest {
      * @param message text to print out
      */
     void log(String message) {
-        System.out.println("%" + whoAmI + "% " + message);
+        System.out.println(whoAmI + message);
     }
 
     void throwFault(String message) {
-        throw new TestFault("%" + whoAmI + "% " + message);
+        throw new TestFault(whoAmI + message);
     }
 
     void throwFault(String message, Throwable t) {
-        throw new TestFault("%" + whoAmI + "% " + message, t);
+        throw new TestFault(whoAmI + message, t);
     }
 
     /**
@@ -116,12 +116,12 @@ public class ShrinkGrowTest {
     public void run() {
         if (System.getProperty("requiresCompressedClassSpace") != null &&
                    !isCompressedClassSpaceAvailable()) {
-                System.out.println("Not applicalbe, Compressed Class Space is required");
+                System.out.println("Not applicable, Compressed Class Space is required");
             return;
         }
 
         try {
-            log("Bootstrapping string concatenation for " + whoAmI );
+            log("Bootstrapping string concatenation");
             go();
             // The quest completed! Yahoo!
             setErrorMessage(null);
@@ -150,7 +150,17 @@ public class ShrinkGrowTest {
             throwFault("We haven't cleaned metaspace yet!");
         } catch (OutOfMemoryError error) {
             if (!isMetaspaceError(error)) {
-                throwFault("Hmm, we ran out metaspace. Metaspace error is still excpected here " + error, error);
+                throwFault("Hmm, we ran out metaspace. Metaspace error is still expected here " + error, error);
+            }
+        } catch(BootstrapMethodError bsme) {
+            Throwable cause = bsme.getCause();
+            if (cause instanceof OutOfMemoryError) {
+                OutOfMemoryError error = (OutOfMemoryError)cause;
+                if (!isMetaspaceError(error)) {
+                    throwFault("Hmm, we got BootstrapMethodError. Metaspace error is still expected as the cause " + error, bsme);
+                }
+            } else {
+                throwFault("We should be out of metaspace but got " + cause, bsme);
             }
         }