8339242: Fix overflow issues in AdlArena

Reviewed-by: jsjolen, kbarrett

src/hotspot/share/adlc/adlArena.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,6 +45,7 @@ void* AdlReAllocateHeap(void* old_ptr, size_t size) {
 }
 
 void* AdlChunk::operator new(size_t requested_size, size_t length) throw() {
+  assert(requested_size <= SIZE_MAX - length, "overflow");
   return AdlCHeapObj::operator new(requested_size + length);
 }
 
@@ -129,6 +130,7 @@ void* AdlArena::grow( size_t x ) {
 //------------------------------calloc-----------------------------------------
 // Allocate zeroed storage in AdlArena
 void *AdlArena::Acalloc( size_t items, size_t x ) {
+  assert(items <= SIZE_MAX / x, "overflow");
   size_t z = items*x;   // Total size needed
   void *ptr = Amalloc(z);       // Get space
   memset( ptr, 0, z );          // Zap space
@@ -136,21 +138,26 @@ void *AdlArena::Acalloc( size_t items, size_t x ) {
 }
 
 //------------------------------realloc----------------------------------------
+static size_t pointer_delta(const void *left, const void *right) {
+  assert(left >= right, "pointer delta underflow");
+  return (uintptr_t)left - (uintptr_t)right;
+}
+
 // Reallocate storage in AdlArena.
 void *AdlArena::Arealloc( void *old_ptr, size_t old_size, size_t new_size ) {
   char *c_old = (char*)old_ptr; // Handy name
-  // Stupid fast special case
-  if( new_size <= old_size ) {  // Shrink in-place
-    if( c_old+old_size == _hwm) // Attempt to free the excess bytes
-      _hwm = c_old+new_size;    // Adjust hwm
-    return c_old;
-  }
 
-  // See if we can resize in-place
-  if( (c_old+old_size == _hwm) &&       // Adjusting recent thing
-      (c_old+new_size <= _max) ) {      // Still fits where it sits
-    _hwm = c_old+new_size;      // Adjust hwm
-    return c_old;               // Return old pointer
+  // Reallocating the latest allocation?
+  if (c_old + old_size == _hwm) {
+    assert(_chunk->bottom() <= c_old, "invariant");
+
+    // Reallocate in place if it fits. Also handles shrinking
+    if (pointer_delta(_max, c_old) >= new_size) {
+      _hwm = c_old + new_size;
+      return c_old;
+    }
+  } else if (new_size <= old_size) { // Shrink in place
+    return c_old;
   }
 
   // Oops, got to relocate guts

src/hotspot/share/adlc/adlArena.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -105,8 +105,10 @@ public:
   // Fast allocate in the arena.  Common case is: pointer test + increment.
   void* Amalloc(size_t x) {
 #ifdef _LP64
+    assert(x <= SIZE_MAX - (8-1), "overflow");
     x = (x + (8-1)) & ((unsigned)(-8));
 #else
+    assert(x <= SIZE_MAX - (4-1), "overflow");
     x = (x + (4-1)) & ((unsigned)(-4));
 #endif
     if (_hwm + x > _max) {

src/hotspot/share/memory/arena.cpp

@@ -311,8 +311,6 @@ void* Arena::grow(size_t x, AllocFailType alloc_failmode) {
   return result;
 }
 
-
-
 // Reallocate storage in Arena.
 void *Arena::Arealloc(void* old_ptr, size_t old_size, size_t new_size, AllocFailType alloc_failmode) {
   if (new_size == 0) {
@@ -324,21 +322,21 @@ void *Arena::Arealloc(void* old_ptr, size_t old_size, size_t new_size, AllocFail
     return Amalloc(new_size, alloc_failmode); // as with realloc(3), a null old ptr is equivalent to malloc(3)
   }
   char *c_old = (char*)old_ptr; // Handy name
-  // Stupid fast special case
-  if( new_size <= old_size ) {  // Shrink in-place
-    if( c_old+old_size == _hwm) // Attempt to free the excess bytes
-      _hwm = c_old+new_size;    // Adjust hwm
-    return c_old;
-  }
 
-  // make sure that new_size is legal
+  // Make sure that new_size is legal
   size_t corrected_new_size = ARENA_ALIGN(new_size);
 
-  // See if we can resize in-place
-  if( (c_old+old_size == _hwm) &&       // Adjusting recent thing
-      (c_old+corrected_new_size <= _max) ) {      // Still fits where it sits
-    _hwm = c_old+corrected_new_size;      // Adjust hwm
-    return c_old;               // Return old pointer
+  // Reallocating the latest allocation?
+  if (c_old + old_size == _hwm) {
+    assert(_chunk->bottom() <= c_old, "invariant");
+
+    // Reallocate in place if it fits. Also handles shrinking
+    if (pointer_delta(_max, c_old, 1) >= corrected_new_size) {
+      _hwm = c_old + corrected_new_size;
+      return c_old;
+    }
+  } else if (new_size <= old_size) { // Shrink in place
+    return c_old;
   }
 
   // Oops, got to relocate guts