8058897: Unsafe.reallocateMemory() ignores -XX:MallocMaxTestWords setting

Reviewed-by: dcubed, dholmes

hotspot/src/share/vm/runtime/os.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -547,20 +547,16 @@ static void verify_memory(void* ptr) {
 // This function supports testing of the malloc out of memory
 // condition without really running the system out of memory.
 //
-static u_char* testMalloc(size_t alloc_size) {
-  assert(MallocMaxTestWords > 0, "sanity check");
+static bool has_reached_max_malloc_test_peak(size_t alloc_size) {
+  if (MallocMaxTestWords > 0) {
+    jint words = (jint)(alloc_size / BytesPerWord);
 
-  if ((cur_malloc_words + (alloc_size / BytesPerWord)) > MallocMaxTestWords) {
-    return NULL;
+    if ((cur_malloc_words + words) > MallocMaxTestWords) {
+      return true;
+    }
+    Atomic::add(words, (volatile jint *)&cur_malloc_words);
   }
-
-  u_char* ptr = (u_char*)::malloc(alloc_size);
-
-  if (ptr != NULL) {
-    Atomic::add(((jint) (alloc_size / BytesPerWord)),
-                (volatile jint *) &cur_malloc_words);
-  }
-  return ptr;
+  return false;
 }
 
 void* os::malloc(size_t size, MEMFLAGS flags) {
@@ -608,13 +604,14 @@ void* os::malloc(size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
 
   NOT_PRODUCT(if (MallocVerifyInterval > 0) check_heap());
 
-  u_char* ptr;
-  if (MallocMaxTestWords > 0) {
-    ptr = testMalloc(alloc_size);
-  } else {
-    ptr = (u_char*)::malloc(alloc_size);
+  // For the test flag -XX:MallocMaxTestWords
+  if (has_reached_max_malloc_test_peak(size)) {
+    return NULL;
   }
 
+  u_char* ptr;
+  ptr = (u_char*)::malloc(alloc_size);
+
 #ifdef ASSERT
   if (ptr == NULL) {
     return NULL;
@@ -642,6 +639,11 @@ void* os::realloc(void *memblock, size_t size, MEMFLAGS flags) {
 
 void* os::realloc(void *memblock, size_t size, MEMFLAGS memflags, const NativeCallStack& stack) {
 
+  // For the test flag -XX:MallocMaxTestWords
+  if (has_reached_max_malloc_test_peak(size)) {
+    return NULL;
+  }
+
 #ifndef ASSERT
   NOT_PRODUCT(inc_stat_counter(&num_mallocs, 1));
   NOT_PRODUCT(inc_stat_counter(&alloc_bytes, size));

hotspot/test/runtime/Unsafe/Reallocate.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8058897
+ * @library /testlibrary
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:MallocMaxTestWords=20m Reallocate
+ */
+
+import com.oracle.java.testlibrary.*;
+import sun.misc.Unsafe;
+import static com.oracle.java.testlibrary.Asserts.*;
+
+public class Reallocate {
+    public static void main(String args[]) throws Exception {
+        Unsafe unsafe = Utils.getUnsafe();
+
+        long address = unsafe.allocateMemory(1);
+        assertNotEquals(address, 0L);
+
+        // Make sure we reallocate correctly
+        unsafe.putByte(address, Byte.MAX_VALUE);
+        address = unsafe.reallocateMemory(address, 2);
+        assertNotEquals(address, 0L);
+        assertEquals(unsafe.getByte(address), Byte.MAX_VALUE);
+
+        // Reallocating with a 0 size should return a null pointer
+        address = unsafe.reallocateMemory(address, 0);
+        assertEquals(address, 0L);
+
+        // Reallocating with a null pointer should result in a normal allocation
+        address = unsafe.reallocateMemory(0L, 1);
+        assertNotEquals(address, 0L);
+        unsafe.putByte(address, Byte.MAX_VALUE);
+        assertEquals(unsafe.getByte(address), Byte.MAX_VALUE);
+
+        // Make sure we can throw an OOME when we fail to reallocate due to OOM
+        try {
+            unsafe.reallocateMemory(address, 20 * 1024 * 1024 * 8);
+        } catch (OutOfMemoryError e) {
+            // Expected
+            return;
+        }
+        throw new RuntimeException("Did not get expected OOM");
+    }
+}