8038268: VM Crashes in MetaspaceShared::generate_vtable_methods while creating CDS archive with limiting SharedMiscCodeSize

Estimate the minimum required size for the misc code region and check if the specified misc code region size meets the minimum size requirement

Reviewed-by: jiangli, dholmes

hotspot/src/share/vm/memory/metaspace.cpp

@@ -3157,6 +3157,16 @@ void Metaspace::global_initialize() {
     SharedMiscDataSize  = align_size_up(SharedMiscDataSize,  max_alignment);
     SharedMiscCodeSize  = align_size_up(SharedMiscCodeSize,  max_alignment);
 
+    // the min_misc_code_size estimate is based on MetaspaceShared::generate_vtable_methods()
+    uintx min_misc_code_size = align_size_up(
+      (MetaspaceShared::num_virtuals * MetaspaceShared::vtbl_list_size) *
+        (sizeof(void*) + MetaspaceShared::vtbl_method_size) + MetaspaceShared::vtbl_common_code_size,
+          max_alignment);
+
+    if (SharedMiscCodeSize < min_misc_code_size) {
+      report_out_of_shared_space(SharedMiscCode);
+    }
+
     // Initialize with the sum of the shared space sizes.  The read-only
     // and read write metaspace chunks will be allocated out of this and the
     // remainder is the misc code and data chunks.

hotspot/src/share/vm/memory/metaspaceShared.hpp

@@ -57,11 +57,16 @@ class MetaspaceShared : AllStatic {
   static bool _archive_loading_failed;
  public:
   enum {
-    vtbl_list_size = 17, // number of entries in the shared space vtable list.
-    num_virtuals = 200   // maximum number of virtual functions
-                         // If virtual functions are added to Metadata,
-                         // this number needs to be increased.  Also,
-                         // SharedMiscCodeSize will need to be increased.
+    vtbl_list_size         = 17,   // number of entries in the shared space vtable list.
+    num_virtuals           = 200,  // maximum number of virtual functions
+                                   // If virtual functions are added to Metadata,
+                                   // this number needs to be increased.  Also,
+                                   // SharedMiscCodeSize will need to be increased.
+                                   // The following 2 sizes were based on
+                                   // MetaspaceShared::generate_vtable_methods()
+    vtbl_method_size       = 16,   // conservative size of the mov1 and jmp instructions
+                                   // for the x64 platform
+    vtbl_common_code_size  = (1*K) // conservative size of the "common_code" for the x64 platform
   };
 
   enum {

hotspot/src/share/vm/utilities/debug.cpp

@@ -256,16 +256,18 @@ void report_out_of_shared_space(SharedSpaceType shared_space) {
   static const char* name[] = {
     "shared read only space",
     "shared read write space",
-    "shared miscellaneous data space"
+    "shared miscellaneous data space",
+    "shared miscellaneous code space"
   };
   static const char* flag[] = {
     "SharedReadOnlySize",
     "SharedReadWriteSize",
-    "SharedMiscDataSize"
+    "SharedMiscDataSize",
+    "SharedMiscCodeSize"
   };
 
    warning("\nThe %s is not large enough\n"
-           "to preload requested classes. Use -XX:%s=\n"
+           "to preload requested classes. Use -XX:%s=<size>\n"
            "to increase the initial size of %s.\n",
            name[shared_space], flag[shared_space], name[shared_space]);
    exit(2);

hotspot/src/share/vm/utilities/debug.hpp

@@ -245,7 +245,8 @@ template <> struct StaticAssert<true> {};
 enum SharedSpaceType {
   SharedReadOnly,
   SharedReadWrite,
-  SharedMiscData
+  SharedMiscData,
+  SharedMiscCode
 };
 
 void report_out_of_shared_space(SharedSpaceType space_type);

hotspot/test/runtime/SharedArchiveFile/LimitSharedSizes.java

@@ -51,9 +51,12 @@ public class LimitSharedSizes {
         // Known issue, JDK-8038422 (assert() on Windows)
         // new SharedSizeTestData("-XX:SharedMiscDataSize", "500k",    "miscellaneous data"),
 
-        // This will cause a VM crash; commenting out for now; see bug JDK-8038268
-        // @ignore JDK-8038268
-        // new SharedSizeTestData("-XX:SharedMiscCodeSize", "20k",     "miscellaneous code"),
+        // Too small of a misc code size should not cause a vm crash.
+        // It should result in the following error message:
+        // The shared miscellaneous code space is not large enough
+        // to preload requested classes. Use -XX:SharedMiscCodeSize=
+        // to increase the initial size of shared miscellaneous code space.
+        new SharedSizeTestData("-XX:SharedMiscCodeSize", "20k",     "miscellaneous code"),
 
         // these values are larger than default ones, but should
         // be acceptable and not cause failure