8293114: JVM should trim the native heap

Reviewed-by: shade, rehn, dholmes

src/hotspot/os/aix/os_aix.inline.hpp

@@ -52,7 +52,7 @@ inline bool os::must_commit_stack_guard_pages() {
 inline void os::map_stack_shadow_pages(address sp) {
 }
 
-// stubbed-out trim-native support
+// Trim-native support, stubbed out for now, may be enabled later
 inline bool os::can_trim_native_heap() { return false; }
 inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; }
 

src/hotspot/os/bsd/os_bsd.inline.hpp

@@ -55,7 +55,7 @@ inline bool os::must_commit_stack_guard_pages() {
 inline void os::map_stack_shadow_pages(address sp) {
 }
 
-// stubbed-out trim-native support
+// Trim-native support, stubbed out for now, may be enabled later
 inline bool os::can_trim_native_heap() { return false; }
 inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; }
 

src/hotspot/os/windows/os_windows.inline.hpp

@@ -93,7 +93,7 @@ inline void PlatformMonitor::notify_all() {
   WakeAllConditionVariable(&_cond);
 }
 
-// stubbed-out trim-native support
+// Trim-native support, stubbed out for now, may be enabled later
 inline bool os::can_trim_native_heap() { return false; }
 inline bool os::trim_native_heap(os::size_change_t* rss_change) { return false; }
 

src/hotspot/share/classfile/stringTable.cpp

@@ -53,6 +53,7 @@
 #include "runtime/mutexLocker.hpp"
 #include "runtime/safepointVerifiers.hpp"
 #include "runtime/timerTrace.hpp"
+#include "runtime/trimNativeHeap.hpp"
 #include "services/diagnosticCommand.hpp"
 #include "utilities/concurrentHashTable.inline.hpp"
 #include "utilities/concurrentHashTableTasks.inline.hpp"
@@ -456,6 +457,7 @@ void StringTable::clean_dead_entries(JavaThread* jt) {
 
   StringTableDeleteCheck stdc;
   StringTableDoDelete stdd;
+  NativeHeapTrimmer::SuspendMark sm("stringtable");
   {
     TraceTime timer("Clean", TRACETIME_LOG(Debug, stringtable, perf));
     while(bdt.do_task(jt, stdc, stdd)) {

src/hotspot/share/classfile/symbolTable.cpp

@@ -37,6 +37,7 @@
 #include "runtime/atomic.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/timerTrace.hpp"
+#include "runtime/trimNativeHeap.hpp"
 #include "services/diagnosticCommand.hpp"
 #include "utilities/concurrentHashTable.inline.hpp"
 #include "utilities/concurrentHashTableTasks.inline.hpp"
@@ -737,6 +738,7 @@ void SymbolTable::clean_dead_entries(JavaThread* jt) {
 
   SymbolTableDeleteCheck stdc;
   SymbolTableDoDelete stdd;
+  NativeHeapTrimmer::SuspendMark sm("symboltable");
   {
     TraceTime timer("Clean", TRACETIME_LOG(Debug, symboltable, perf));
     while (bdt.do_task(jt, stdc, stdd)) {

src/hotspot/share/logging/logTag.hpp

@@ -198,6 +198,7 @@ class outputStream;
   LOG_TAG(timer) \
   LOG_TAG(tlab) \
   LOG_TAG(tracking) \
+  LOG_TAG(trimnative) /* trim native heap */ \
   LOG_TAG(unload) /* Trace unloading of classes */ \
   LOG_TAG(unmap) \
   LOG_TAG(unshareable) \

src/hotspot/share/memory/arena.cpp

@@ -30,6 +30,7 @@
 #include "runtime/os.hpp"
 #include "runtime/task.hpp"
 #include "runtime/threadCritical.hpp"
+#include "runtime/trimNativeHeap.hpp"
 #include "services/memTracker.inline.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
@@ -92,6 +93,7 @@ class ChunkPool {
   }
 
   static void clean() {
+    NativeHeapTrimmer::SuspendMark sm("chunk pool cleaner");
     for (int i = 0; i < _num_pools; i++) {
       _pools[i].prune();
     }

src/hotspot/share/prims/whitebox.cpp

@@ -2587,6 +2587,14 @@ WB_ENTRY(jboolean, WB_SetVirtualThreadsNotifyJvmtiMode(JNIEnv* env, jobject wb,
   return result;
 WB_END
 
+WB_ENTRY(void, WB_PreTouchMemory(JNIEnv* env, jobject wb, jlong addr, jlong size))
+  void* const from = (void*)addr;
+  void* const to = (void*)(addr + size);
+  if (from > to) {
+    os::pretouch_memory(from, to, os::vm_page_size());
+  }
+WB_END
+
 #define CC (char*)
 
 static JNINativeMethod methods[] = {
@@ -2869,6 +2877,7 @@ static JNINativeMethod methods[] = {
   {CC"lockCritical",    CC"()V",                      (void*)&WB_LockCritical},
   {CC"unlockCritical",  CC"()V",                      (void*)&WB_UnlockCritical},
   {CC"setVirtualThreadsNotifyJvmtiMode", CC"(Z)Z",    (void*)&WB_SetVirtualThreadsNotifyJvmtiMode},
+  {CC"preTouchMemory",  CC"(JJ)V",                    (void*)&WB_PreTouchMemory},
 };
 
 

src/hotspot/share/runtime/globals.hpp

@@ -1985,6 +1985,13 @@ const int ObjectAlignmentInBytes = 8;
           "1: monitors & legacy stack-locking (LM_LEGACY, default), "       \
           "2: monitors & new lightweight locking (LM_LIGHTWEIGHT)")         \
           range(0, 2)                                                       \
+                                                                            \
+  product(uint, TrimNativeHeapInterval, 0, EXPERIMENTAL,                    \
+          "Interval, in ms, at which the JVM will trim the native heap if " \
+          "the platform supports that. Lower values will reclaim memory "   \
+          "more eagerly at the cost of higher overhead. A value of 0 "      \
+          "(default) disables native heap trimming.")                       \
+          range(0, UINT_MAX)                                                \
 
 // end of RUNTIME_FLAGS
 

src/hotspot/share/runtime/java.cpp

@@ -70,6 +70,7 @@
 #include "runtime/task.hpp"
 #include "runtime/threads.hpp"
 #include "runtime/timer.hpp"
+#include "runtime/trimNativeHeap.hpp"
 #include "runtime/vmOperations.hpp"
 #include "runtime/vmThread.hpp"
 #include "runtime/vm_version.hpp"
@@ -477,6 +478,8 @@ void before_exit(JavaThread* thread, bool halt) {
   StatSampler::disengage();
   StatSampler::destroy();
 
+  NativeHeapTrimmer::cleanup();
+
   // Stop concurrent GC threads
   Universe::heap()->stop();
 

src/hotspot/share/runtime/synchronizer.cpp

@@ -54,6 +54,7 @@
 #include "runtime/synchronizer.hpp"
 #include "runtime/threads.hpp"
 #include "runtime/timer.hpp"
+#include "runtime/trimNativeHeap.hpp"
 #include "runtime/vframe.hpp"
 #include "runtime/vmThread.hpp"
 #include "utilities/align.hpp"
@@ -1646,6 +1647,7 @@ public:
 };
 
 static size_t delete_monitors(GrowableArray<ObjectMonitor*>* delete_list) {
+  NativeHeapTrimmer::SuspendMark sm("monitor deletion");
   size_t count = 0;
   for (ObjectMonitor* monitor: *delete_list) {
     delete monitor;

src/hotspot/share/runtime/threads.cpp

@@ -87,6 +87,7 @@
 #include "runtime/threadSMR.inline.hpp"
 #include "runtime/timer.hpp"
 #include "runtime/timerTrace.hpp"
+#include "runtime/trimNativeHeap.hpp"
 #include "runtime/vmOperations.hpp"
 #include "runtime/vm_version.hpp"
 #include "services/attachListener.hpp"
@@ -759,6 +760,10 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
   }
 #endif
 
+  if (NativeHeapTrimmer::enabled()) {
+    NativeHeapTrimmer::initialize();
+  }
+
   // Always call even when there are not JVMTI environments yet, since environments
   // may be attached late and JVMTI must track phases of VM execution
   JvmtiExport::enter_live_phase();

src/hotspot/share/runtime/trimNativeHeap.cpp

@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2023 Red Hat Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/globals_extension.hpp"
+#include "runtime/mutex.hpp"
+#include "runtime/mutexLocker.hpp"
+#include "runtime/nonJavaThread.hpp"
+#include "runtime/os.inline.hpp"
+#include "runtime/safepoint.hpp"
+#include "runtime/trimNativeHeap.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/ostream.hpp"
+#include "utilities/vmError.hpp"
+
+class NativeHeapTrimmerThread : public NamedThread {
+
+  // Upper limit for the backoff during pending/in-progress safepoint.
+  // Chosen as reasonable value to balance the overheads of waking up
+  // during the safepoint, which might have undesired effects on latencies,
+  // and the accuracy in tracking the trimming interval.
+  static constexpr int64_t safepoint_poll_ms = 250;
+
+  Monitor* const _lock;
+  bool _stop;
+  uint16_t _suspend_count;
+
+  // Statistics
+  uint64_t _num_trims_performed;
+
+  bool is_suspended() const {
+    assert(_lock->is_locked(), "Must be");
+    return _suspend_count > 0;
+  }
+
+  uint16_t inc_suspend_count() {
+    assert(_lock->is_locked(), "Must be");
+    assert(_suspend_count < UINT16_MAX, "Sanity");
+    return ++_suspend_count;
+  }
+
+  uint16_t dec_suspend_count() {
+    assert(_lock->is_locked(), "Must be");
+    assert(_suspend_count != 0, "Sanity");
+    return --_suspend_count;
+  }
+
+  bool is_stopped() const {
+    assert(_lock->is_locked(), "Must be");
+    return _stop;
+  }
+
+  bool at_or_nearing_safepoint() const {
+    return SafepointSynchronize::is_at_safepoint() ||
+           SafepointSynchronize::is_synchronizing();
+  }
+
+  // in seconds
+  static double now() { return os::elapsedTime(); }
+  static double to_ms(double seconds) { return seconds * 1000.0; }
+
+  struct LogStartStopMark {
+    void log(const char* s) { log_info(trimnative)("Native heap trimmer %s", s); }
+    LogStartStopMark()  { log("start"); }
+    ~LogStartStopMark() { log("stop"); }
+  };
+
+  void run() override {
+    assert(NativeHeapTrimmer::enabled(), "Only call if enabled");
+
+    LogStartStopMark lssm;
+
+    const double interval_secs = (double)TrimNativeHeapInterval / 1000;
+
+    while (true) {
+      double tnow = now();
+      double next_trim_time = tnow + interval_secs;
+
+      unsigned times_suspended = 0;
+      unsigned times_waited = 0;
+      unsigned times_safepoint = 0;
+
+      {
+        MonitorLocker ml(_lock, Mutex::_no_safepoint_check_flag);
+        if (_stop) return;
+
+        while (at_or_nearing_safepoint() || is_suspended() || next_trim_time > tnow) {
+          if (is_suspended()) {
+            times_suspended ++;
+            ml.wait(0); // infinite
+          } else if (next_trim_time > tnow) {
+            times_waited ++;
+            const int64_t wait_ms = MAX2(1.0, to_ms(next_trim_time - tnow));
+            ml.wait(wait_ms);
+          } else if (at_or_nearing_safepoint()) {
+            times_safepoint ++;
+            const int64_t wait_ms = MIN2<int64_t>(TrimNativeHeapInterval, safepoint_poll_ms);
+            ml.wait(wait_ms);
+          }
+
+          if (_stop) return;
+
+          tnow = now();
+        }
+      }
+
+      log_trace(trimnative)("Times: %u suspended, %u timed, %u safepoint",
+                            times_suspended, times_waited, times_safepoint);
+
+      execute_trim_and_log(tnow);
+    }
+  }
+
+  // Execute the native trim, log results.
+  void execute_trim_and_log(double t1) {
+    assert(os::can_trim_native_heap(), "Unexpected");
+
+    os::size_change_t sc = { 0, 0 };
+    LogTarget(Info, trimnative) lt;
+    const bool logging_enabled = lt.is_enabled();
+
+    // We only collect size change information if we are logging; save the access to procfs otherwise.
+    if (os::trim_native_heap(logging_enabled ? &sc : nullptr)) {
+      _num_trims_performed++;
+      if (logging_enabled) {
+        double t2 = now();
+        if (sc.after != SIZE_MAX) {
+          const size_t delta = sc.after < sc.before ? (sc.before - sc.after) : (sc.after - sc.before);
+          const char sign = sc.after < sc.before ? '-' : '+';
+          log_info(trimnative)("Periodic Trim (" UINT64_FORMAT "): " PROPERFMT "->" PROPERFMT " (%c" PROPERFMT ") %.3fms",
+                               _num_trims_performed,
+                               PROPERFMTARGS(sc.before), PROPERFMTARGS(sc.after), sign, PROPERFMTARGS(delta),
+                               to_ms(t2 - t1));
+        } else {
+          log_info(trimnative)("Periodic Trim (" UINT64_FORMAT "): complete (no details) %.3fms",
+                               _num_trims_performed,
+                               to_ms(t2 - t1));
+        }
+      }
+    }
+  }
+
+public:
+
+  NativeHeapTrimmerThread() :
+    _lock(new (std::nothrow) PaddedMonitor(Mutex::nosafepoint, "NativeHeapTrimmer_lock")),
+    _stop(false),
+    _suspend_count(0),
+    _num_trims_performed(0)
+  {
+    set_name("Native Heap Trimmer");
+    if (os::create_thread(this, os::vm_thread)) {
+      os::start_thread(this);
+    }
+  }
+
+  void suspend(const char* reason) {
+    assert(NativeHeapTrimmer::enabled(), "Only call if enabled");
+    uint16_t n = 0;
+    {
+      MonitorLocker ml(_lock, Mutex::_no_safepoint_check_flag);
+      n = inc_suspend_count();
+      // No need to wakeup trimmer
+    }
+    log_debug(trimnative)("Trim suspended for %s (%u suspend requests)", reason, n);
+  }
+
+  void resume(const char* reason) {
+    assert(NativeHeapTrimmer::enabled(), "Only call if enabled");
+    uint16_t n = 0;
+    {
+      MonitorLocker ml(_lock, Mutex::_no_safepoint_check_flag);
+      n = dec_suspend_count();
+      if (n == 0) {
+        ml.notify_all(); // pause end
+      }
+    }
+    if (n == 0) {
+      log_debug(trimnative)("Trim resumed after %s", reason);
+    } else {
+      log_debug(trimnative)("Trim still suspended after %s (%u suspend requests)", reason, n);
+    }
+  }
+
+  void stop() {
+    MonitorLocker ml(_lock, Mutex::_no_safepoint_check_flag);
+    _stop = true;
+    ml.notify_all();
+  }
+
+  void print_state(outputStream* st) const {
+    // Don't pull lock during error reporting
+    Mutex* const lock = VMError::is_error_reported() ? nullptr : _lock;
+    int64_t num_trims = 0;
+    bool stopped = false;
+    uint16_t suspenders = 0;
+    {
+      MutexLocker ml(lock, Mutex::_no_safepoint_check_flag);
+      num_trims = _num_trims_performed;
+      stopped = _stop;
+      suspenders = _suspend_count;
+    }
+    st->print_cr("Trims performed: " UINT64_FORMAT ", current suspend count: %d, stopped: %d",
+                 num_trims, suspenders, stopped);
+  }
+
+}; // NativeHeapTrimmer
+
+static NativeHeapTrimmerThread* g_trimmer_thread = nullptr;
+
+void NativeHeapTrimmer::initialize() {
+  assert(g_trimmer_thread == nullptr, "Only once");
+  if (TrimNativeHeapInterval > 0) {
+    if (!os::can_trim_native_heap()) {
+      FLAG_SET_ERGO(TrimNativeHeapInterval, 0);
+      log_warning(trimnative)("Native heap trim is not supported on this platform");
+      return;
+    }
+    g_trimmer_thread = new NativeHeapTrimmerThread();
+    log_info(trimnative)("Periodic native trim enabled (interval: %u ms)", TrimNativeHeapInterval);
+  }
+}
+
+void NativeHeapTrimmer::cleanup() {
+  if (g_trimmer_thread != nullptr) {
+    g_trimmer_thread->stop();
+  }
+}
+
+void NativeHeapTrimmer::suspend_periodic_trim(const char* reason) {
+  if (g_trimmer_thread != nullptr) {
+    g_trimmer_thread->suspend(reason);
+  }
+}
+
+void NativeHeapTrimmer::resume_periodic_trim(const char* reason) {
+  if (g_trimmer_thread != nullptr) {
+    g_trimmer_thread->resume(reason);
+  }
+}
+
+void NativeHeapTrimmer::print_state(outputStream* st) {
+  if (g_trimmer_thread != nullptr) {
+    st->print_cr("Periodic native trim enabled (interval: %u ms)", TrimNativeHeapInterval);
+    g_trimmer_thread->print_state(st);
+  } else {
+    st->print_cr("Periodic native trim disabled");
+  }
+}

src/hotspot/share/runtime/trimNativeHeap.hpp

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2023 Red Hat Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_RUNTIME_TRIMNATIVEHEAP_HPP
+#define SHARE_RUNTIME_TRIMNATIVEHEAP_HPP
+
+#include "memory/allStatic.hpp"
+#include "runtime/globals.hpp"
+
+class outputStream;
+
+class NativeHeapTrimmer : public AllStatic {
+
+  // Pause periodic trim (if enabled).
+  static void suspend_periodic_trim(const char* reason);
+
+  // Unpause periodic trim (if enabled).
+  static void resume_periodic_trim(const char* reason);
+
+public:
+
+  static void initialize();
+  static void cleanup();
+
+  static inline bool enabled() { return TrimNativeHeapInterval > 0; }
+
+  static void print_state(outputStream* st);
+
+  // Pause periodic trimming while in scope; when leaving scope,
+  // resume periodic trimming.
+  struct SuspendMark {
+    const char* const _reason;
+    SuspendMark(const char* reason = "unknown") : _reason(reason) {
+      if (NativeHeapTrimmer::enabled()) {
+        suspend_periodic_trim(_reason);
+      }
+    }
+    ~SuspendMark()  {
+      if (NativeHeapTrimmer::enabled()) {
+        resume_periodic_trim(_reason);
+      }
+    }
+  };
+};
+
+#endif // SHARE_RUNTIME_TRIMNATIVEHEAP_HPP

src/hotspot/share/utilities/vmError.cpp

@@ -54,6 +54,7 @@
 #include "runtime/stackOverflow.hpp"
 #include "runtime/threads.hpp"
 #include "runtime/threadSMR.hpp"
+#include "runtime/trimNativeHeap.hpp"
 #include "runtime/vmThread.hpp"
 #include "runtime/vmOperations.hpp"
 #include "runtime/vm_version.hpp"
@@ -1289,9 +1290,13 @@ void VMError::report(outputStream* st, bool _verbose) {
 
   STEP_IF("Native Memory Tracking", _verbose)
     MemTracker::error_report(st);
+    st->cr();
+
+  STEP_IF("printing periodic trim state", _verbose)
+    NativeHeapTrimmer::print_state(st);
+    st->cr();
 
   STEP_IF("printing system", _verbose)
-    st->cr();
     st->print_cr("---------------  S Y S T E M  ---------------");
     st->cr();
 
@@ -1458,10 +1463,14 @@ void VMError::print_vm_info(outputStream* st) {
   // STEP("Native Memory Tracking")
 
   MemTracker::error_report(st);
+  st->cr();
+
+  // STEP("printing periodic trim state")
+  NativeHeapTrimmer::print_state(st);
+  st->cr();
+
 
   // STEP("printing system")
-
-  st->cr();
   st->print_cr("---------------  S Y S T E M  ---------------");
   st->cr();
 

test/hotspot/gtest/runtime/test_trim_native.cpp

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2023 Red Hat Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "runtime/os.hpp"
+#include "runtime/trimNativeHeap.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/ostream.hpp"
+#include "testutils.hpp"
+#include "unittest.hpp"
+
+using ::testing::HasSubstr;
+
+// Check the state of the trimmer via print_state; returns the suspend count
+static int check_trim_state() {
+  char buf [1024];
+  stringStream ss(buf, sizeof(buf));
+  NativeHeapTrimmer::print_state(&ss);
+  if (NativeHeapTrimmer::enabled()) {
+    assert(TrimNativeHeapInterval > 0, "Sanity");
+    EXPECT_THAT(buf, HasSubstr("Periodic native trim enabled"));
+
+    const char* s = ::strstr(buf, "Trims performed");
+    EXPECT_NOT_NULL(s);
+
+    uint64_t num_trims = 0;
+    int suspend_count = 0;
+    int stopped = 0;
+    EXPECT_EQ(::sscanf(s, "Trims performed: " UINT64_FORMAT ", current suspend count: %d, stopped: %d",
+                       &num_trims, &suspend_count, &stopped), 3);
+
+    // Number of trims we can reasonably expect should be limited
+    const double fudge_factor = 1.5;
+    const uint64_t elapsed_ms = (uint64_t)(os::elapsedTime() * fudge_factor * 1000.0);
+    const uint64_t max_num_trims = (elapsed_ms / TrimNativeHeapInterval) + 1;
+    EXPECT_LE(num_trims, max_num_trims);
+
+    // We should not be stopped
+    EXPECT_EQ(stopped, 0);
+
+    // Suspend count must not underflow
+    EXPECT_GE(suspend_count, 0);
+    return suspend_count;
+
+  } else {
+    EXPECT_THAT(buf, HasSubstr("Periodic native trim disabled"));
+    EXPECT_THAT(buf, Not(HasSubstr("Trims performed")));
+    return 0;
+  }
+}
+
+TEST_VM(os, TrimNative) {
+
+  if (!NativeHeapTrimmer::enabled()) {
+    return;
+  }
+
+  // Try recursive pausing. This tests that we are able to pause, that pauses stack,
+  // and that stacking works within the same thread.
+  int c1 = 0, c2 = 0, c3 = 0;
+  {
+    NativeHeapTrimmer::SuspendMark sm1("Test1");
+    c1 = check_trim_state();
+    {
+      NativeHeapTrimmer::SuspendMark sm2("Test2");
+      c2 = check_trim_state();
+      {
+        NativeHeapTrimmer::SuspendMark sm3("Test3");
+        c3 = check_trim_state();
+      }
+    }
+  }
+  // We also check the state: the suspend count should go up. But since we don't know
+  // whether concurrent code will have increased the suspend count too, this is fuzzy and
+  // we must avoid intermittent false positives.
+  EXPECT_GT(c2, c1);
+  EXPECT_GT(c3, c2);
+}

test/hotspot/jtreg/gtest/NativeHeapTrimmerGtest.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023 Red Hat, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/* @test
+ * @summary Run a subset of gtests with the native trimmer activated.
+ * @library /test/lib
+ * @modules java.base/jdk.internal.misc
+ *          java.xml
+ * @run main/native GTestWrapper --gtest_filter=os.trim* -Xlog:trimnative -XX:+UnlockExperimentalVMOptions -XX:TrimNativeHeapInterval=100
+ */

test/hotspot/jtreg/runtime/os/TestTrimNative.java

@@ -0,0 +1,309 @@
+/*
+ * Copyright (c) 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2023 Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test id=trimNative
+ * @requires (os.family=="linux") & !vm.musl
+ * @modules java.base/jdk.internal.misc
+ * @library /test/lib
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run driver TestTrimNative trimNative
+ */
+
+/*
+ * @test id=trimNativeHighInterval
+ * @summary High interval trimming should not even kick in for short program runtimes
+ * @requires (os.family=="linux") & !vm.musl
+ * @modules java.base/jdk.internal.misc
+ * @library /test/lib
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run driver TestTrimNative trimNativeHighInterval
+ */
+
+/*
+ * @test id=trimNativeLowInterval
+ * @summary Very low (sub-second) interval, nothing should explode
+ * @requires (os.family=="linux") & !vm.musl
+ * @modules java.base/jdk.internal.misc
+ * @library /test/lib
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run driver TestTrimNative trimNativeLowInterval
+ */
+
+/*
+ * @test id=testOffByDefault
+ * @summary Test that trimming is disabled by default
+ * @requires (os.family=="linux") & !vm.musl
+ * @modules java.base/jdk.internal.misc
+ * @library /test/lib
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run driver TestTrimNative testOffByDefault
+ */
+
+/*
+ * @test id=testOffExplicit
+ * @summary Test that trimming can be disabled explicitly
+ * @requires (os.family=="linux") & !vm.musl
+ * @modules java.base/jdk.internal.misc
+ * @library /test/lib
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run driver TestTrimNative testOffExplicit
+ */
+
+/*
+ * @test id=testOffOnNonCompliantPlatforms
+ * @summary Test that trimming is correctly reported as unavailable if unavailable
+ * @requires (os.family!="linux") | vm.musl
+ * @modules java.base/jdk.internal.misc
+ * @library /test/lib
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run driver TestTrimNative testOffOnNonCompliantPlatforms
+ */
+
+import jdk.test.lib.Platform;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import jdk.test.whitebox.WhiteBox;
+
+public class TestTrimNative {
+
+    // Actual RSS increase is a lot larger than 4 MB. Depends on glibc overhead, and NMT malloc headers in debug VMs.
+    // We need small-grained allocations to make sure they actually increase RSS (all touched) and to see the
+    // glibc-retaining-memory effect.
+    static final int szAllocations = 128;
+    static final int totalAllocationsSize = 128 * 1024 * 1024; // 128 MB total
+    static final int numAllocations = totalAllocationsSize / szAllocations;
+
+    static long[] ptrs = new long[numAllocations];
+
+    enum Unit {
+        B(1), K(1024), M(1024*1024), G(1024*1024*1024);
+        public final long size;
+        Unit(long size) { this.size = size; }
+    }
+
+    private static String[] prepareOptions(String[] extraVMOptions, String[] programOptions) {
+        List<String> allOptions = new ArrayList<String>();
+        if (extraVMOptions != null) {
+            allOptions.addAll(Arrays.asList(extraVMOptions));
+        }
+        allOptions.add("-Xmx128m");
+        allOptions.add("-Xms128m"); // Stabilize RSS
+        allOptions.add("-XX:+AlwaysPreTouch"); // Stabilize RSS
+        allOptions.add("-XX:+UnlockDiagnosticVMOptions"); // For whitebox
+        allOptions.add("-XX:+WhiteBoxAPI");
+        allOptions.add("-Xbootclasspath/a:.");
+        allOptions.add("-XX:-ExplicitGCInvokesConcurrent"); // Invoke explicit GC on System.gc
+        allOptions.add("-Xlog:trimnative=debug");
+        allOptions.add("--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED");
+        if (programOptions != null) {
+            allOptions.addAll(Arrays.asList(programOptions));
+        }
+        return allOptions.toArray(new String[0]);
+    }
+
+    private static OutputAnalyzer runTestWithOptions(String[] extraOptions, String[] programOptions) throws IOException {
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(prepareOptions(extraOptions, programOptions));
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+        output.shouldHaveExitValue(0);
+        return output;
+    }
+
+    private static void checkExpectedLogMessages(OutputAnalyzer output, boolean expectEnabled,
+                                                 int expectedInterval) {
+        if (expectEnabled) {
+            output.shouldContain("Periodic native trim enabled (interval: " + expectedInterval + " ms");
+            output.shouldContain("Native heap trimmer start");
+            output.shouldContain("Native heap trimmer stop");
+        } else {
+            output.shouldNotContain("Periodic native trim enabled");
+        }
+    }
+
+    /**
+     * Given JVM output, look for one or more log lines that describes a successful negative trim. The total amount
+     * of trims should be matching about what the test program allocated.
+     * @param output
+     * @param minTrimsExpected min number of periodic trim lines expected in UL log
+     * @param maxTrimsExpected min number of periodic trim lines expected in UL log
+     */
+    private static void parseOutputAndLookForNegativeTrim(OutputAnalyzer output, int minTrimsExpected,
+                                                          int maxTrimsExpected) {
+        output.reportDiagnosticSummary();
+        List<String> lines = output.asLines();
+        Pattern pat = Pattern.compile(".*\\[trimnative\\] Periodic Trim \\(\\d+\\): (\\d+)([BKMG])->(\\d+)([BKMG]).*");
+        int numTrimsFound = 0;
+        long rssReductionTotal = 0;
+        for (String line : lines) {
+            Matcher mat = pat.matcher(line);
+            if (mat.matches()) {
+                long rss1 = Long.parseLong(mat.group(1)) * Unit.valueOf(mat.group(2)).size;
+                long rss2 = Long.parseLong(mat.group(3)) * Unit.valueOf(mat.group(4)).size;
+                if (rss1 > rss2) {
+                    rssReductionTotal += (rss1 - rss2);
+                }
+                numTrimsFound ++;
+            }
+            if (numTrimsFound > maxTrimsExpected) {
+                throw new RuntimeException("Abnormal high number of periodic trim attempts found (more than " + maxTrimsExpected +
+                        "). Does the interval setting not work?");
+            }
+        }
+        if (numTrimsFound < minTrimsExpected) {
+            throw new RuntimeException("We found fewer (periodic) trim lines in UL log than expected (expected at least " + minTrimsExpected +
+                    ", found " + numTrimsFound + ").");
+        }
+        if (maxTrimsExpected > 0) {
+            // This is very fuzzy. Test program malloced X bytes, then freed them again and trimmed. But the log line prints change in RSS.
+            // Which, of course, is influenced by a lot of other factors. But we expect to see *some* reasonable reduction in RSS
+            // due to trimming.
+            float fudge = 0.5f;
+            // On ppc, we see a vastly diminished return (~3M reduction instead of ~200), I suspect because of the underlying
+            // 64k pages lead to a different geometry. Manual tests with larger reclaim sizes show that autotrim works. For
+            // this test, we just reduce the fudge factor.
+            if (Platform.isPPC()) { // le and be both
+                fudge = 0.01f;
+            }
+            long expectedMinimalReduction = (long) (totalAllocationsSize * fudge);
+            if (rssReductionTotal < expectedMinimalReduction) {
+                throw new RuntimeException("We did not see the expected RSS reduction in the UL log. Expected (with fudge)" +
+                        " to see at least a combined reduction of " + expectedMinimalReduction + ".");
+            }
+        }
+    }
+
+    static class Tester {
+        public static void main(String[] args) throws Exception {
+            long sleeptime = Long.parseLong(args[0]);
+
+            System.out.println("Will spike now...");
+            WhiteBox wb = WhiteBox.getWhiteBox();
+            for (int i = 0; i < numAllocations; i++) {
+                ptrs[i] = wb.NMTMalloc(szAllocations);
+                wb.preTouchMemory(ptrs[i], szAllocations);
+            }
+            for (int i = 0; i < numAllocations; i++) {
+                wb.NMTFree(ptrs[i]);
+            }
+            System.out.println("Done spiking.");
+
+            System.out.println("GC...");
+            System.gc();
+
+            // give GC time to react
+            System.out.println("Sleeping...");
+            Thread.sleep(sleeptime);
+            System.out.println("Done.");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+
+        if (args.length == 0) {
+            throw new RuntimeException("Argument error");
+        }
+
+        switch (args[0]) {
+            case "trimNative": {
+                long trimInterval = 500; // twice per second
+                long ms1 = System.currentTimeMillis();
+                OutputAnalyzer output = runTestWithOptions(
+                        new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:TrimNativeHeapInterval=" + trimInterval },
+                        new String[] { TestTrimNative.Tester.class.getName(), "5000" }
+                );
+                long ms2 = System.currentTimeMillis();
+                long runtime_ms = ms2 - ms1;
+
+                checkExpectedLogMessages(output, true, 500);
+
+                long maxTrimsExpected = runtime_ms / trimInterval;
+                long minTrimsExpected = maxTrimsExpected / 2;
+                parseOutputAndLookForNegativeTrim(output, (int) minTrimsExpected, (int) maxTrimsExpected);
+            } break;
+
+            case "trimNativeHighInterval": {
+                OutputAnalyzer output = runTestWithOptions(
+                        new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:TrimNativeHeapInterval=" + Integer.MAX_VALUE },
+                        new String[] { TestTrimNative.Tester.class.getName(), "5000" }
+                );
+                checkExpectedLogMessages(output, true, Integer.MAX_VALUE);
+                // We should not see any trims since the interval would prevent them
+                parseOutputAndLookForNegativeTrim(output, 0, 0);
+            } break;
+
+            case "trimNativeLowInterval": {
+                OutputAnalyzer output = runTestWithOptions(
+                        new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:TrimNativeHeapInterval=1" },
+                        new String[] { TestTrimNative.Tester.class.getName(), "0" }
+                );
+                checkExpectedLogMessages(output, true, 1);
+                parseOutputAndLookForNegativeTrim(output, 1, 3000);
+            } break;
+
+            case "testOffOnNonCompliantPlatforms": {
+                OutputAnalyzer output = runTestWithOptions(
+                        new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:TrimNativeHeapInterval=1" },
+                        new String[] { "-version" }
+                );
+                checkExpectedLogMessages(output, false, 0);
+                parseOutputAndLookForNegativeTrim(output, 0, 0);
+                // The following output is expected to be printed with warning level, so it should not need -Xlog
+                output.shouldContain("[warning][trimnative] Native heap trim is not supported on this platform");
+            } break;
+
+            case "testOffExplicit": {
+                OutputAnalyzer output = runTestWithOptions(
+                        new String[] { "-XX:+UnlockExperimentalVMOptions", "-XX:TrimNativeHeapInterval=0" },
+                        new String[] { "-version" }
+                );
+                checkExpectedLogMessages(output, false, 0);
+                parseOutputAndLookForNegativeTrim(output, 0, 0);
+            } break;
+
+            case "testOffByDefault": {
+                OutputAnalyzer output = runTestWithOptions(null, new String[] { "-version" } );
+                checkExpectedLogMessages(output, false, 0);
+                parseOutputAndLookForNegativeTrim(output, 0, 0);
+            } break;
+
+            default:
+                throw new RuntimeException("Invalid test " + args[0]);
+
+        }
+    }
+}

test/hotspot/jtreg/serviceability/dcmd/vm/TrimLibcHeapTest.java

@@ -22,6 +22,7 @@
  * questions.
  */
 
+import jdk.test.lib.Platform;
 import org.testng.annotations.Test;
 import jdk.test.lib.dcmd.CommandExecutor;
 import jdk.test.lib.dcmd.JMXExecutor;
@@ -31,7 +32,7 @@ import jdk.test.lib.process.OutputAnalyzer;
  * @test
  * @summary Test of diagnostic command VM.trim_libc_heap
  * @library /test/lib
- * @requires (os.family=="linux") & !vm.musl
+ * @requires os.family == "linux"
  * @modules java.base/jdk.internal.misc
  *          java.compiler
  *          java.management
@@ -42,7 +43,11 @@ public class TrimLibcHeapTest {
     public void run(CommandExecutor executor) {
         OutputAnalyzer output = executor.execute("System.trim_native_heap");
         output.reportDiagnosticSummary();
-        output.shouldMatch(".*Trim native heap: RSS\\+Swap: \\d+[BKM]->\\d+[BKM].*");
+        if (Platform.isMusl()) {
+            output.shouldContain("Not available");
+        } else {
+            output.shouldMatch("Trim native heap: RSS\\+Swap: \\d+[BKMG]->\\d+[BKMG] \\(-\\d+[BKMG]\\)");
+        }
     }
 
     @Test

test/lib/jdk/test/whitebox/WhiteBox.java

@@ -767,4 +767,6 @@ public class WhiteBox {
   public native void unlockCritical();
 
   public native boolean setVirtualThreadsNotifyJvmtiMode(boolean enabled);
+
+  public native void preTouchMemory(long addr, long size);
 }