8055289: Internal Error: mallocTracker.cpp:146 fatal error: Should not use malloc for big memory block, use virtual memory instead

Return NULL if memory allocation size is bigger than MAX_MALLOC_SIZE when NMT is on

Reviewed-by: coleenp, gtriantafill

hotspot/src/share/vm/runtime/os.cpp

@@ -53,6 +53,7 @@
 #include "runtime/vm_version.hpp"
 #include "services/attachListener.hpp"
 #include "services/nmtCommon.hpp"
+#include "services/mallocTracker.hpp"
 #include "services/memTracker.hpp"
 #include "services/threadService.hpp"
 #include "utilities/defaultStream.hpp"
@@ -570,6 +571,17 @@ void* os::malloc(size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
   NOT_PRODUCT(inc_stat_counter(&num_mallocs, 1));
   NOT_PRODUCT(inc_stat_counter(&alloc_bytes, size));
 
+#if INCLUDE_NMT
+  // NMT can not track malloc allocation size > MAX_MALLOC_SIZE, which is
+  // (1GB - 1) on 32-bit system. It is not an issue on 64-bit system, where
+  // MAX_MALLOC_SIZE = ((1 << 62) - 1).
+  // VM code does not have such large malloc allocation. However, it can come
+  // Unsafe call.
+  if (MemTracker::tracking_level() >= NMT_summary && size > MAX_MALLOC_SIZE) {
+    return NULL;
+  }
+#endif
+
 #ifdef ASSERT
   // checking for the WatcherThread and crash_protection first
   // since os::malloc can be called when the libjvm.{dll,so} is
@@ -640,6 +652,13 @@ void* os::realloc(void *memblock, size_t size, MEMFLAGS flags) {
 }
 
 void* os::realloc(void *memblock, size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
+#if INCLUDE_NMT
+  // See comments in os::malloc() above
+  if (MemTracker::tracking_level() >= NMT_summary && size > MAX_MALLOC_SIZE) {
+    return NULL;
+  }
+#endif
+
 #ifndef ASSERT
   NOT_PRODUCT(inc_stat_counter(&num_mallocs, 1));
   NOT_PRODUCT(inc_stat_counter(&alloc_bytes, size));

hotspot/test/TEST.groups

@@ -87,6 +87,7 @@ needs_jdk = \
   runtime/NMT/SummarySanityCheck.java \
   runtime/NMT/ThreadedMallocTestType.java \
   runtime/NMT/ThreadedVirtualAllocTestType.java \
+  runtime/NMT/UnsafeMallocLimit.java \
   runtime/NMT/VirtualAllocCommitUncommitRecommit.java \
   runtime/NMT/VirtualAllocTestType.java \
   runtime/RedefineObject/TestRedefineObject.java \

hotspot/test/runtime/NMT/UnsafeMallocLimit.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8055289
+ * @library /testlibrary
+ * @build UnsafeMallocLimit
+ * @run main/othervm -Xmx32m -XX:NativeMemoryTracking=summary UnsafeMallocLimit
+ */
+
+import com.oracle.java.testlibrary.*;
+import sun.misc.Unsafe;
+
+public class UnsafeMallocLimit {
+
+    public static void main(String args[]) throws Exception {
+        if (Platform.is32bit()) {
+            Unsafe unsafe = Utils.getUnsafe();
+            try {
+                unsafe.allocateMemory(1 << 30);
+                throw new RuntimeException("Did not get expected OOME");
+            } catch (OutOfMemoryError e) {
+                // Expected exception
+            }
+        } else {
+            System.out.println("Test only valid on 32-bit platforms");
+        }
+    }
+}