8199067: [REDO] NMT: Enhance thread stack tracking

Precise thread stack tracking on Linux and Windows

Reviewed-by: stuefe, coleenp

src/hotspot/os/linux/os_linux.cpp

@@ -3111,6 +3111,68 @@ static address get_stack_commited_bottom(address bottom, size_t size) {
   return nbot;
 }
 
+bool os::committed_in_range(address start, size_t size, address& committed_start, size_t& committed_size) {
+  int mincore_return_value;
+  const size_t stripe = 1024;  // query this many pages each time
+  unsigned char vec[stripe];
+  const size_t page_sz = os::vm_page_size();
+  size_t pages = size / page_sz;
+
+  assert(is_aligned(start, page_sz), "Start address must be page aligned");
+  assert(is_aligned(size, page_sz), "Size must be page aligned");
+
+  committed_start = NULL;
+
+  int loops = (pages + stripe - 1) / stripe;
+  int committed_pages = 0;
+  address loop_base = start;
+  for (int index = 0; index < loops; index ++) {
+    assert(pages > 0, "Nothing to do");
+    int pages_to_query = (pages >= stripe) ? stripe : pages;
+    pages -= pages_to_query;
+
+    // Get stable read
+    while ((mincore_return_value = mincore(loop_base, pages_to_query * page_sz, vec)) == -1 && errno == EAGAIN);
+
+    // During shutdown, some memory goes away without properly notifying NMT,
+    // E.g. ConcurrentGCThread/WatcherThread can exit without deleting thread object.
+    // Bailout and return as not committed for now.
+    if (mincore_return_value == -1 && errno == ENOMEM) {
+      return false;
+    }
+
+    assert(mincore_return_value == 0, "Range must be valid");
+    // Process this stripe
+    for (int vecIdx = 0; vecIdx < pages_to_query; vecIdx ++) {
+      if ((vec[vecIdx] & 0x01) == 0) { // not committed
+        // End of current contiguous region
+        if (committed_start != NULL) {
+          break;
+        }
+      } else { // committed
+        // Start of region
+        if (committed_start == NULL) {
+          committed_start = loop_base + page_sz * vecIdx;
+        }
+        committed_pages ++;
+      }
+    }
+
+    loop_base += pages_to_query * page_sz;
+  }
+
+  if (committed_start != NULL) {
+    assert(committed_pages > 0, "Must have committed region");
+    assert(committed_pages <= int(size / page_sz), "Can not commit more than it has");
+    assert(committed_start >= start && committed_start < start + size, "Out of range");
+    committed_size = page_sz * committed_pages;
+    return true;
+  } else {
+    assert(committed_pages == 0, "Should not have committed region");
+    return false;
+  }
+}
+
 
 // Linux uses a growable mapping for the stack, and if the mapping for
 // the stack guard pages is not removed when we detach a thread the

src/hotspot/os/windows/os_windows.cpp

@@ -365,6 +365,39 @@ size_t os::current_stack_size() {
   return sz;
 }
 
+bool os::committed_in_range(address start, size_t size, address& committed_start, size_t& committed_size) {
+  MEMORY_BASIC_INFORMATION minfo;
+  committed_start = NULL;
+  committed_size = 0;
+  address top = start + size;
+  const address start_addr = start;
+  while (start < top) {
+    VirtualQuery(start, &minfo, sizeof(minfo));
+    if ((minfo.State & MEM_COMMIT) == 0) {  // not committed
+      if (committed_start != NULL) {
+        break;
+      }
+    } else {  // committed
+      if (committed_start == NULL) {
+        committed_start = start;
+      }
+      size_t offset = start - (address)minfo.BaseAddress;
+      committed_size += minfo.RegionSize - offset;
+    }
+    start = (address)minfo.BaseAddress + minfo.RegionSize;
+  }
+
+  if (committed_start == NULL) {
+    assert(committed_size == 0, "Sanity");
+    return false;
+  } else {
+    assert(committed_start >= start_addr && committed_start < top, "Out of range");
+    // current region may go beyond the limit, trim to the limit
+    committed_size = MIN2(committed_size, size_t(top - committed_start));
+    return true;
+  }
+}
+
 struct tm* os::localtime_pd(const time_t* clock, struct tm* res) {
   const struct tm* time_struct_ptr = localtime(clock);
   if (time_struct_ptr != NULL) {

src/hotspot/share/runtime/os.cpp

@@ -251,6 +251,14 @@ bool os::dll_build_name(char* buffer, size_t size, const char* fname) {
   return (n != -1);
 }
 
+#if !defined(LINUX) && !defined(_WINDOWS)
+bool os::committed_in_range(address start, size_t size, address& committed_start, size_t& committed_size) {
+  committed_start = start;
+  committed_size = size;
+  return true;
+}
+#endif
+
 // Helper for dll_locate_lib.
 // Pass buffer and printbuffer as we already printed the path to buffer
 // when we called get_current_directory. This way we avoid another buffer

src/hotspot/share/runtime/os.hpp

@@ -273,6 +273,10 @@ class os: AllStatic {
   static void map_stack_shadow_pages(address sp);
   static bool stack_shadow_pages_available(Thread *thread, const methodHandle& method, address sp);
 
+  // Find committed memory region within specified range (start, start + size),
+  // return true if found any
+  static bool committed_in_range(address start, size_t size, address& committed_start, size_t& committed_size);
+
   // OS interface to Virtual Memory
 
   // Return the default page size.

src/hotspot/share/services/memTracker.hpp

@@ -246,7 +246,7 @@ class MemTracker : AllStatic {
     if (addr != NULL) {
       // uses thread stack malloc slot for book keeping number of threads
       MallocMemorySummary::record_malloc(0, mtThreadStack);
-      record_virtual_memory_reserve_and_commit(addr, size, CALLER_PC, mtThreadStack);
+      record_virtual_memory_reserve(addr, size, CALLER_PC, mtThreadStack);
     }
   }
 

src/hotspot/share/services/virtualMemoryTracker.cpp

@@ -23,6 +23,7 @@
  */
 #include "precompiled.hpp"
 
+#include "logging/log.hpp"
 #include "memory/metaspace.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/os.hpp"
@@ -38,6 +39,12 @@ void VirtualMemorySummary::initialize() {
   ::new ((void*)_snapshot) VirtualMemorySnapshot();
 }
 
+void VirtualMemorySummary::snapshot(VirtualMemorySnapshot* s) {
+  // Snapshot current thread stacks
+  VirtualMemoryTracker::snapshot_thread_stacks();
+  as_snapshot()->copy_to(s);
+}
+
 SortedLinkedList<ReservedMemoryRegion, compare_reserved_region_base>* VirtualMemoryTracker::_reserved_regions;
 
 int compare_committed_region(const CommittedMemoryRegion& r1, const CommittedMemoryRegion& r2) {
@@ -286,6 +293,26 @@ void ReservedMemoryRegion::set_flag(MEMFLAGS f) {
   }
 }
 
+address ReservedMemoryRegion::thread_stack_uncommitted_bottom() const {
+  assert(flag() == mtThreadStack, "Only for thread stack");
+  LinkedListNode<CommittedMemoryRegion>* head = _committed_regions.head();
+  address bottom = base();
+  address top = base() + size();
+  while (head != NULL) {
+    address committed_top = head->data()->base() + head->data()->size();
+    if (committed_top < top) {
+      // committed stack guard pages, skip them
+      bottom = head->data()->base() + head->data()->size();
+      head = head->next();
+    } else {
+      assert(top == committed_top, "Sanity");
+      break;
+    }
+  }
+
+  return bottom;
+}
+
 bool VirtualMemoryTracker::initialize(NMT_TrackingLevel level) {
   if (level >= NMT_summary) {
     VirtualMemorySummary::initialize();
@@ -460,6 +487,80 @@ bool VirtualMemoryTracker::remove_released_region(address addr, size_t size) {
   }
 }
 
+// Iterate the range, find committed region within its bound.
+class RegionIterator : public StackObj {
+private:
+  const address _start;
+  const size_t  _size;
+
+  address _current_start;
+  size_t  _current_size;
+public:
+  RegionIterator(address start, size_t size) :
+    _start(start), _size(size), _current_start(start), _current_size(size) {
+  }
+
+  // return true if committed region is found
+  bool next_committed(address& start, size_t& size);
+private:
+  address end() const { return _start + _size; }
+};
+
+bool RegionIterator::next_committed(address& committed_start, size_t& committed_size) {
+  if (end() <= _current_start) return false;
+
+  const size_t page_sz = os::vm_page_size();
+  assert(_current_start + _current_size == end(), "Must be");
+  if (os::committed_in_range(_current_start, _current_size, committed_start, committed_size)) {
+    assert(committed_start != NULL, "Must be");
+    assert(committed_size > 0 && is_aligned(committed_size, os::vm_page_size()), "Must be");
+
+    size_t remaining_size = (_current_start + _current_size) - (committed_start + committed_size);
+    _current_start = committed_start + committed_size;
+    _current_size = remaining_size;
+    return true;
+  } else {
+    return false;
+  }
+}
+
+// Walk all known thread stacks, snapshot their committed ranges.
+class SnapshotThreadStackWalker : public VirtualMemoryWalker {
+public:
+  SnapshotThreadStackWalker() {}
+
+  bool do_allocation_site(const ReservedMemoryRegion* rgn) {
+    if (rgn->flag() == mtThreadStack) {
+      address stack_bottom = rgn->thread_stack_uncommitted_bottom();
+      address committed_start;
+      size_t  committed_size;
+      size_t stack_size = rgn->base() + rgn->size() - stack_bottom;
+
+      ReservedMemoryRegion* region = const_cast<ReservedMemoryRegion*>(rgn);
+      NativeCallStack ncs; // empty stack
+
+      RegionIterator itr(stack_bottom, stack_size);
+      DEBUG_ONLY(bool found_stack = false;)
+      while (itr.next_committed(committed_start, committed_size)) {
+        assert(committed_start != NULL, "Should not be null");
+        assert(committed_size > 0, "Should not be 0");
+        region->add_committed_region(committed_start, committed_size, ncs);
+        DEBUG_ONLY(found_stack = true;)
+      }
+#ifdef ASSERT
+      if (!found_stack) {
+        log_debug(thread)("Thread exited without proper cleanup, may leak thread object");
+      }
+#endif
+    }
+    return true;
+  }
+};
+
+void VirtualMemoryTracker::snapshot_thread_stacks() {
+  SnapshotThreadStackWalker walker;
+  walk_virtual_memory(&walker);
+}
 
 bool VirtualMemoryTracker::walk_virtual_memory(VirtualMemoryWalker* walker) {
   assert(_reserved_regions != NULL, "Sanity check");

src/hotspot/share/services/virtualMemoryTracker.hpp

@@ -160,9 +160,7 @@ class VirtualMemorySummary : AllStatic {
     as_snapshot()->by_type(to)->commit_memory(size);
   }
 
-  static inline void snapshot(VirtualMemorySnapshot* s) {
-    as_snapshot()->copy_to(s);
-  }
+  static void snapshot(VirtualMemorySnapshot* s);
 
   static VirtualMemorySnapshot* as_snapshot() {
     return (VirtualMemorySnapshot*)_snapshot;
@@ -336,6 +334,9 @@ class ReservedMemoryRegion : public VirtualMemoryRegion {
     return compare(rgn) == 0;
   }
 
+  // uncommitted thread stack bottom, above guard pages if there is any.
+  address thread_stack_uncommitted_bottom() const;
+
   bool    add_committed_region(address addr, size_t size, const NativeCallStack& stack);
   bool    remove_uncommitted_region(address addr, size_t size);
 
@@ -389,6 +390,7 @@ class VirtualMemoryWalker : public StackObj {
 // Main class called from MemTracker to track virtual memory allocations, commits and releases.
 class VirtualMemoryTracker : AllStatic {
   friend class VirtualMemoryTrackerTest;
+  friend class CommittedVirtualMemoryTest;
 
  public:
   static bool initialize(NMT_TrackingLevel level);
@@ -408,6 +410,9 @@ class VirtualMemoryTracker : AllStatic {
 
   static bool transition(NMT_TrackingLevel from, NMT_TrackingLevel to);
 
+  // Snapshot current thread stacks
+  static void snapshot_thread_stacks();
+
  private:
   static SortedLinkedList<ReservedMemoryRegion, compare_reserved_region_base>* _reserved_regions;
 };

test/hotspot/gtest/runtime/test_committed_virtualmemory.cpp

@@ -0,0 +1,210 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+
+// Included early because the NMT flags don't include it.
+#include "utilities/macros.hpp"
+
+#include "runtime/thread.hpp"
+#include "services/memTracker.hpp"
+#include "services/virtualMemoryTracker.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "unittest.hpp"
+
+
+class CommittedVirtualMemoryTest {
+public:
+  static void test() {
+    Thread* thr = Thread::current();
+    address stack_end = thr->stack_end();
+    size_t  stack_size = thr->stack_size();
+
+    MemTracker::record_thread_stack(stack_end, stack_size);
+
+    VirtualMemoryTracker::add_reserved_region(stack_end, stack_size, CALLER_PC, mtThreadStack);
+
+    // snapshot current stack usage
+    VirtualMemoryTracker::snapshot_thread_stacks();
+
+    ReservedMemoryRegion* rmr = VirtualMemoryTracker::_reserved_regions->find(ReservedMemoryRegion(stack_end, stack_size));
+    ASSERT_TRUE(rmr != NULL);
+
+    ASSERT_EQ(rmr->base(), stack_end);
+    ASSERT_EQ(rmr->size(), stack_size);
+
+    CommittedRegionIterator iter = rmr->iterate_committed_regions();
+    int i = 0;
+    address i_addr = (address)&i;
+    bool found_i_addr = false;
+
+    // stack grows downward
+    address stack_top = stack_end + stack_size;
+    bool found_stack_top = false;
+
+    for (const CommittedMemoryRegion* region = iter.next(); region != NULL; region = iter.next()) {
+      if (region->base() + region->size() == stack_top) {
+        ASSERT_TRUE(region->size() <= stack_size);
+        found_stack_top = true;
+      }
+
+      if(i_addr < stack_top && i_addr >= region->base()) {
+        found_i_addr = true;
+      }
+
+      i++;
+    }
+
+    // stack and guard pages may be contiguous as one region
+    ASSERT_TRUE(i >= 1);
+    ASSERT_TRUE(found_stack_top);
+    ASSERT_TRUE(found_i_addr);
+  }
+
+  static void check_covered_pages(address addr, size_t size, address base, size_t touch_pages, int* page_num) {
+    const size_t page_sz = os::vm_page_size();
+    size_t index;
+    for (index = 0; index < touch_pages; index ++) {
+      address page_addr = base + page_num[index] * page_sz;
+      // The range covers this page, marks the page
+      if (page_addr >= addr && page_addr < addr + size) {
+        page_num[index] = -1;
+      }
+    }
+  }
+
+  static void test_committed_region_impl(size_t num_pages, size_t touch_pages, int* page_num) {
+    const size_t page_sz = os::vm_page_size();
+    const size_t size = num_pages * page_sz;
+    char* base = os::reserve_memory(size, NULL, page_sz, mtThreadStack);
+    bool result = os::commit_memory(base, size, false);
+    size_t index;
+    ASSERT_NE(base, (char*)NULL);
+    for (index = 0; index < touch_pages; index ++) {
+      char* touch_addr = base + page_sz * page_num[index];
+      *touch_addr = 'a';
+    }
+
+    address frame = (address)0x1235;
+    NativeCallStack stack(&frame, 1);
+    VirtualMemoryTracker::add_reserved_region((address)base, size, stack, mtThreadStack);
+
+    // trigger the test
+    VirtualMemoryTracker::snapshot_thread_stacks();
+
+    ReservedMemoryRegion* rmr = VirtualMemoryTracker::_reserved_regions->find(ReservedMemoryRegion((address)base, size));
+    ASSERT_TRUE(rmr != NULL);
+
+    bool precise_tracking_supported = false;
+    CommittedRegionIterator iter = rmr->iterate_committed_regions();
+    for (const CommittedMemoryRegion* region = iter.next(); region != NULL; region = iter.next()) {
+      if (region->size() == size) {
+        // platforms that do not support precise tracking.
+        ASSERT_TRUE(iter.next() == NULL);
+        break;
+      } else {
+        precise_tracking_supported = true;
+        check_covered_pages(region->base(), region->size(), (address)base, touch_pages, page_num);
+      }
+    }
+
+    if (precise_tracking_supported) {
+      // All touched pages should be committed
+      for (size_t index = 0; index < touch_pages; index ++) {
+        ASSERT_EQ(page_num[index], -1);
+      }
+    }
+
+    // Cleanup
+    os::free_memory(base, size, page_sz);
+    VirtualMemoryTracker::remove_released_region((address)base, size);
+
+    rmr = VirtualMemoryTracker::_reserved_regions->find(ReservedMemoryRegion((address)base, size));
+    ASSERT_TRUE(rmr == NULL);
+  }
+
+  static void test_committed_region() {
+    // On Linux, we scan 1024 pages at a time.
+    // Here, we test scenario that scans < 1024 pages.
+    int small_range[] = {3, 9, 46};
+    int mid_range[] = {0, 45, 100, 399, 400, 1000, 1031};
+    int large_range[] = {100, 301, 1024, 2047, 2048, 2049, 2050, 3000};
+
+    test_committed_region_impl(47, 3, small_range);
+    test_committed_region_impl(1088, 5, mid_range);
+    test_committed_region_impl(3074, 8, large_range);
+  }
+
+  static void test_partial_region() {
+    bool   result;
+    size_t committed_size;
+    address committed_start;
+    size_t index;
+
+    const size_t page_sz = os::vm_page_size();
+    const size_t num_pages = 4;
+    const size_t size = num_pages * page_sz;
+    char* base = os::reserve_memory(size, NULL, page_sz, mtTest);
+    ASSERT_NE(base, (char*)NULL);
+    result = os::commit_memory(base, size, false);
+
+    ASSERT_TRUE(result);
+    // touch all pages
+    for (index = 0; index < num_pages; index ++) {
+      *(base + index * page_sz) = 'a';
+    }
+
+    // Test whole range
+    result = os::committed_in_range((address)base, size, committed_start, committed_size);
+    ASSERT_TRUE(result);
+    ASSERT_EQ(num_pages * page_sz, committed_size);
+    ASSERT_EQ(committed_start, (address)base);
+
+    // Test beginning of the range
+    result = os::committed_in_range((address)base, 2 * page_sz, committed_start, committed_size);
+    ASSERT_TRUE(result);
+    ASSERT_EQ(2 * page_sz, committed_size);
+    ASSERT_EQ(committed_start, (address)base);
+
+    // Test end of the range
+    result = os::committed_in_range((address)(base + page_sz), 3 * page_sz, committed_start, committed_size);
+    ASSERT_TRUE(result);
+    ASSERT_EQ(3 * page_sz, committed_size);
+    ASSERT_EQ(committed_start, (address)(base + page_sz));
+
+    // Test middle of the range
+    result = os::committed_in_range((address)(base + page_sz), 2 * page_sz, committed_start, committed_size);
+    ASSERT_TRUE(result);
+    ASSERT_EQ(2 * page_sz, committed_size);
+    ASSERT_EQ(committed_start, (address)(base + page_sz));
+  }
+};
+
+TEST_VM(CommittedVirtualMemoryTracker, test_committed_virtualmemory_region) {
+  VirtualMemoryTracker::initialize(NMT_detail);
+  VirtualMemoryTracker::late_initialize(NMT_detail);
+
+  CommittedVirtualMemoryTest::test();
+  CommittedVirtualMemoryTest::test_committed_region();
+  CommittedVirtualMemoryTest::test_partial_region();
+}