8260267: ZGC: Reduce mark stack usage

Co-authored-by: Wang Chao <wchao@openjdk.org>
Co-authored-by: Hui Shi <hshi@openjdk.org>
Reviewed-by: sjohanss, ayang

src/hotspot/share/gc/z/zBarrier.cpp

@@ -26,6 +26,7 @@
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zOop.inline.hpp"
+#include "gc/z/zThread.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/safepoint.hpp"
@@ -64,7 +65,7 @@ bool ZBarrier::should_mark_through(uintptr_t addr) {
   return true;
 }
 
-template <bool follow, bool finalizable, bool publish>
+template <bool gc_thread, bool follow, bool finalizable, bool publish>
 uintptr_t ZBarrier::mark(uintptr_t addr) {
   uintptr_t good_addr;
 
@@ -81,7 +82,7 @@ uintptr_t ZBarrier::mark(uintptr_t addr) {
 
   // Mark
   if (should_mark_through<finalizable>(addr)) {
-    ZHeap::heap()->mark_object<follow, finalizable, publish>(good_addr);
+    ZHeap::heap()->mark_object<gc_thread, follow, finalizable, publish>(good_addr);
   }
 
   if (finalizable) {
@@ -111,11 +112,11 @@ uintptr_t ZBarrier::relocate(uintptr_t addr) {
 }
 
 uintptr_t ZBarrier::relocate_or_mark(uintptr_t addr) {
-  return during_relocate() ? relocate(addr) : mark<Follow, Strong, Publish>(addr);
+  return during_relocate() ? relocate(addr) : mark<AnyThread, Follow, Strong, Publish>(addr);
 }
 
 uintptr_t ZBarrier::relocate_or_mark_no_follow(uintptr_t addr) {
-  return during_relocate() ? relocate(addr) : mark<DontFollow, Strong, Publish>(addr);
+  return during_relocate() ? relocate(addr) : mark<AnyThread, DontFollow, Strong, Publish>(addr);
 }
 
 uintptr_t ZBarrier::relocate_or_remap(uintptr_t addr) {
@@ -169,6 +170,13 @@ uintptr_t ZBarrier::weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr) {
 //
 // Keep alive barrier
 //
+uintptr_t ZBarrier::keep_alive_barrier_on_oop_slow_path(uintptr_t addr) {
+  assert(during_mark(), "Invalid phase");
+
+  // Mark
+  return mark<AnyThread, Follow, Strong, Overflow>(addr);
+}
+
 uintptr_t ZBarrier::keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr) {
   const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
   assert(ZHeap::heap()->is_object_strongly_live(good_addr), "Should be live");
@@ -186,16 +194,18 @@ uintptr_t ZBarrier::keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr)
 //
 uintptr_t ZBarrier::mark_barrier_on_oop_slow_path(uintptr_t addr) {
   assert(during_mark(), "Invalid phase");
+  assert(ZThread::is_worker(), "Invalid thread");
 
   // Mark
-  return mark<Follow, Strong, Overflow>(addr);
+  return mark<GCThread, Follow, Strong, Overflow>(addr);
 }
 
 uintptr_t ZBarrier::mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr) {
   assert(during_mark(), "Invalid phase");
+  assert(ZThread::is_worker(), "Invalid thread");
 
   // Mark
-  return mark<Follow, Finalizable, Overflow>(addr);
+  return mark<GCThread, Follow, Finalizable, Overflow>(addr);
 }
 
 //

src/hotspot/share/gc/z/zBarrier.hpp

@@ -33,6 +33,9 @@ typedef uintptr_t (*ZBarrierSlowPath)(uintptr_t);
 
 class ZBarrier : public AllStatic {
 private:
+  static const bool GCThread    = true;
+  static const bool AnyThread   = false;
+
   static const bool Follow      = true;
   static const bool DontFollow  = false;
 
@@ -55,7 +58,7 @@ private:
   static bool during_mark();
   static bool during_relocate();
   template <bool finalizable> static bool should_mark_through(uintptr_t addr);
-  template <bool follow, bool finalizable, bool publish> static uintptr_t mark(uintptr_t addr);
+  template <bool gc_thread, bool follow, bool finalizable, bool publish> static uintptr_t mark(uintptr_t addr);
   static uintptr_t remap(uintptr_t addr);
   static uintptr_t relocate(uintptr_t addr);
   static uintptr_t relocate_or_mark(uintptr_t addr);
@@ -69,6 +72,7 @@ private:
   static uintptr_t weak_load_barrier_on_weak_oop_slow_path(uintptr_t addr);
   static uintptr_t weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr);
 
+  static uintptr_t keep_alive_barrier_on_oop_slow_path(uintptr_t addr);
   static uintptr_t keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr);
   static uintptr_t keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr);
 
@@ -104,10 +108,10 @@ public:
   static bool is_alive_barrier_on_phantom_oop(oop o);
 
   // Keep alive barrier
+  static void keep_alive_barrier_on_oop(oop o);
   static void keep_alive_barrier_on_weak_oop_field(volatile oop* p);
   static void keep_alive_barrier_on_phantom_oop_field(volatile oop* p);
   static void keep_alive_barrier_on_phantom_root_oop_field(oop* p);
-  static void keep_alive_barrier_on_oop(oop o);
 
   // Mark barrier
   static void mark_barrier_on_oop_field(volatile oop* p, bool finalizable);

src/hotspot/share/gc/z/zBarrier.inline.hpp

@@ -364,7 +364,7 @@ inline void ZBarrier::keep_alive_barrier_on_oop(oop o) {
   assert(ZAddress::is_good(addr), "Invalid address");
 
   if (during_mark()) {
-    mark_barrier_on_oop_slow_path(addr);
+    keep_alive_barrier_on_oop_slow_path(addr);
   }
 }
 

src/hotspot/share/gc/z/zDriver.cpp

@@ -44,6 +44,7 @@ static const ZStatPhaseCycle      ZPhaseCycle("Garbage Collection Cycle");
 static const ZStatPhasePause      ZPhasePauseMarkStart("Pause Mark Start");
 static const ZStatPhaseConcurrent ZPhaseConcurrentMark("Concurrent Mark");
 static const ZStatPhaseConcurrent ZPhaseConcurrentMarkContinue("Concurrent Mark Continue");
+static const ZStatPhaseConcurrent ZPhaseConcurrentMarkFree("Concurrent Mark Free");
 static const ZStatPhasePause      ZPhasePauseMarkEnd("Pause Mark End");
 static const ZStatPhaseConcurrent ZPhaseConcurrentProcessNonStrongReferences("Concurrent Process Non-Strong References");
 static const ZStatPhaseConcurrent ZPhaseConcurrentResetRelocationSet("Concurrent Reset Relocation Set");
@@ -322,6 +323,11 @@ void ZDriver::concurrent_mark_continue() {
   ZHeap::heap()->mark(false /* initial */);
 }
 
+void ZDriver::concurrent_mark_free() {
+  ZStatTimer timer(ZPhaseConcurrentMarkFree);
+  ZHeap::heap()->mark_free();
+}
+
 void ZDriver::concurrent_process_non_strong_references() {
   ZStatTimer timer(ZPhaseConcurrentProcessNonStrongReferences);
   ZBreakpoint::at_after_reference_processing_started();
@@ -426,22 +432,25 @@ void ZDriver::gc(GCCause::Cause cause) {
     concurrent(mark_continue);
   }
 
-  // Phase 4: Concurrent Process Non-Strong References
+  // Phase 4: Concurrent Mark Free
+  concurrent(mark_free);
+
+  // Phase 5: Concurrent Process Non-Strong References
   concurrent(process_non_strong_references);
 
-  // Phase 5: Concurrent Reset Relocation Set
+  // Phase 6: Concurrent Reset Relocation Set
   concurrent(reset_relocation_set);
 
-  // Phase 6: Pause Verify
+  // Phase 7: Pause Verify
   pause_verify();
 
-  // Phase 7: Concurrent Select Relocation Set
+  // Phase 8: Concurrent Select Relocation Set
   concurrent(select_relocation_set);
 
-  // Phase 8: Pause Relocate Start
+  // Phase 9: Pause Relocate Start
   pause_relocate_start();
 
-  // Phase 9: Concurrent Relocate
+  // Phase 10: Concurrent Relocate
   concurrent(relocate);
 }
 

src/hotspot/share/gc/z/zDriver.hpp

@@ -41,6 +41,7 @@ private:
   void concurrent_mark();
   bool pause_mark_end();
   void concurrent_mark_continue();
+  void concurrent_mark_free();
   void concurrent_process_non_strong_references();
   void concurrent_reset_relocation_set();
   void pause_verify();

src/hotspot/share/gc/z/zHeap.cpp

@@ -284,6 +284,10 @@ bool ZHeap::mark_end() {
   return true;
 }
 
+void ZHeap::mark_free() {
+  _mark.free();
+}
+
 void ZHeap::keep_alive(oop obj) {
   ZBarrier::keep_alive_barrier_on_oop(obj);
 }

src/hotspot/share/gc/z/zHeap.hpp

@@ -122,11 +122,12 @@ public:
   // Marking
   bool is_object_live(uintptr_t addr) const;
   bool is_object_strongly_live(uintptr_t addr) const;
-  template <bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
+  template <bool gc_thread, bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
   void mark_start();
   void mark(bool initial);
   void mark_flush_and_free(Thread* thread);
   bool mark_end();
+  void mark_free();
   void keep_alive(oop obj);
 
   // Relocation set

src/hotspot/share/gc/z/zHeap.inline.hpp

@@ -57,10 +57,10 @@ inline bool ZHeap::is_object_strongly_live(uintptr_t addr) const {
   return page->is_object_strongly_live(addr);
 }
 
-template <bool follow, bool finalizable, bool publish>
+template <bool gc_thread, bool follow, bool finalizable, bool publish>
 inline void ZHeap::mark_object(uintptr_t addr) {
   assert(ZGlobalPhase == ZPhaseMark, "Mark not allowed");
-  _mark.mark_object<follow, finalizable, publish>(addr);
+  _mark.mark_object<gc_thread, follow, finalizable, publish>(addr);
 }
 
 inline uintptr_t ZHeap::alloc_tlab(size_t size) {

src/hotspot/share/gc/z/zMark.cpp

@@ -278,28 +278,6 @@ void ZMark::follow_object(oop obj, bool finalizable) {
   }
 }
 
-bool ZMark::try_mark_object(ZMarkCache* cache, uintptr_t addr, bool finalizable) {
-  ZPage* const page = _page_table->get(addr);
-  if (page->is_allocating()) {
-    // Newly allocated objects are implicitly marked
-    return false;
-  }
-
-  // Try mark object
-  bool inc_live = false;
-  const bool success = page->mark_object(addr, finalizable, inc_live);
-  if (inc_live) {
-    // Update live objects/bytes for page. We use the aligned object
-    // size since that is the actual number of bytes used on the page
-    // and alignment paddings can never be reclaimed.
-    const size_t size = ZUtils::object_size(addr);
-    const size_t aligned_size = align_up(size, page->object_alignment());
-    cache->inc_live(page, aligned_size);
-  }
-
-  return success;
-}
-
 void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) {
   // Decode flags
   const bool finalizable = entry.finalizable();
@@ -310,24 +288,38 @@ void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) {
     return;
   }
 
-  // Decode object address and follow flag
+  // Decode object address and additional flags
   const uintptr_t addr = entry.object_address();
+  const bool mark = entry.mark();
+  bool inc_live = entry.inc_live();
+  const bool follow = entry.follow();
 
-  if (!try_mark_object(cache, addr, finalizable)) {
+  ZPage* const page = _page_table->get(addr);
+  assert(page->is_relocatable(), "Invalid page state");
+
+  // Mark
+  if (mark && !page->mark_object(addr, finalizable, inc_live)) {
     // Already marked
     return;
   }
 
-  if (is_array(addr)) {
-    // Decode follow flag
-    const bool follow = entry.follow();
+  // Increment live
+  if (inc_live) {
+    // Update live objects/bytes for page. We use the aligned object
+    // size since that is the actual number of bytes used on the page
+    // and alignment paddings can never be reclaimed.
+    const size_t size = ZUtils::object_size(addr);
+    const size_t aligned_size = align_up(size, page->object_alignment());
+    cache->inc_live(page, aligned_size);
+  }
 
-    // The follow flag is currently only relevant for object arrays
-    if (follow) {
+  // Follow
+  if (follow) {
+    if (is_array(addr)) {
       follow_array_object(objArrayOop(ZOop::from_address(addr)), finalizable);
+    } else {
+      follow_object(ZOop::from_address(addr), finalizable);
     }
-  } else {
-    follow_object(ZOop::from_address(addr), finalizable);
   }
 }
 
@@ -776,6 +768,14 @@ bool ZMark::end() {
   return true;
 }
 
+void ZMark::free() {
+  // Free any unused mark stack space
+  _allocator.free();
+
+  // Update statistics
+  ZStatMark::set_at_mark_free(_allocator.size());
+}
+
 void ZMark::flush_and_free() {
   Thread* const thread = Thread::current();
   flush_and_free(thread);

src/hotspot/share/gc/z/zMark.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -64,7 +64,6 @@ private:
   void follow_partial_array(ZMarkStackEntry entry, bool finalizable);
   void follow_array_object(objArrayOop obj, bool finalizable);
   void follow_object(oop obj, bool finalizable);
-  bool try_mark_object(ZMarkCache* cache, uintptr_t addr, bool finalizable);
   void mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry);
 
   template <typename T> bool drain(ZMarkStripe* stripe,
@@ -101,11 +100,12 @@ public:
 
   bool is_initialized() const;
 
-  template <bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
+  template <bool gc_thread, bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
 
   void start();
   void mark(bool initial);
   bool end();
+  void free();
 
   void flush_and_free();
   bool flush_and_free(Thread* thread);

src/hotspot/share/gc/z/zMark.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,17 +27,52 @@
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zMark.hpp"
 #include "gc/z/zMarkStack.inline.hpp"
+#include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageTable.inline.hpp"
 #include "gc/z/zThreadLocalData.hpp"
 #include "runtime/thread.hpp"
 #include "utilities/debug.hpp"
 
-template <bool follow, bool finalizable, bool publish>
+// Marking before pushing helps reduce mark stack memory usage. However,
+// we only mark before pushing in GC threads to avoid burdening Java threads
+// with writing to, and potentially first having to clear, mark bitmaps.
+//
+// It's also worth noting that while marking an object can be done at any
+// time in the marking phase, following an object can only be done after
+// root processing has called ClassLoaderDataGraph::clear_claimed_marks(),
+// since it otherwise would interact badly with claiming of CLDs.
+
+template <bool gc_thread, bool follow, bool finalizable, bool publish>
 inline void ZMark::mark_object(uintptr_t addr) {
   assert(ZAddress::is_marked(addr), "Should be marked");
+
+  ZPage* const page = _page_table->get(addr);
+  if (page->is_allocating()) {
+    // Already implicitly marked
+    return;
+  }
+
+  const bool mark_before_push = gc_thread;
+  bool inc_live = false;
+
+  if (mark_before_push) {
+    // Try mark object
+    if (!page->mark_object(addr, finalizable, inc_live)) {
+      // Already marked
+      return;
+    }
+  } else {
+    // Don't push if already marked
+    if (page->is_object_marked<finalizable>(addr)) {
+      // Already marked
+      return;
+    }
+  }
+
+  // Push
   ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
   ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
-  ZMarkStackEntry entry(addr, follow, finalizable);
-
+  ZMarkStackEntry entry(addr, !mark_before_push, inc_live, follow, finalizable);
   stacks->push(&_allocator, &_stripes, stripe, entry, publish);
 }
 

src/hotspot/share/gc/z/zMarkStack.hpp

@@ -64,6 +64,8 @@ public:
 
   void push(T* stack);
   T* pop();
+
+  void clear();
 };
 
 using ZMarkStack = ZStack<ZMarkStackEntry, ZMarkStackSlots>;

src/hotspot/share/gc/z/zMarkStack.inline.hpp

@@ -156,6 +156,11 @@ inline T* ZStackList<T>::pop() {
   }
 }
 
+template <typename T>
+inline void ZStackList<T>::clear() {
+  _head = encode_versioned_pointer(NULL, 0);
+}
+
 inline bool ZMarkStripe::is_empty() const {
   return _published.is_empty() && _overflowed.is_empty();
 }

src/hotspot/share/gc/z/zMarkStackAllocator.cpp

@@ -54,12 +54,63 @@ ZMarkStackSpace::ZMarkStackSpace() :
 
   // Register mark stack space start
   ZMarkStackSpaceStart = _start;
+
+  // Prime space
+  _end += expand_space();
 }
 
 bool ZMarkStackSpace::is_initialized() const {
   return _start != 0;
 }
 
+size_t ZMarkStackSpace::size() const {
+  return _end - _start;
+}
+
+size_t ZMarkStackSpace::used() const {
+  return _top - _start;
+}
+
+size_t ZMarkStackSpace::expand_space() {
+  const size_t expand_size = ZMarkStackSpaceExpandSize;
+  const size_t old_size = size();
+  const size_t new_size = old_size + expand_size;
+
+  if (new_size > ZMarkStackSpaceLimit) {
+    // Expansion limit reached. This is a fatal error since we
+    // currently can't recover from running out of mark stack space.
+    fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
+          "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
+          ZMarkStackSpaceLimit / M);
+  }
+
+  log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
+                         old_size / M, new_size / M);
+
+  // Expand
+  os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
+
+  return expand_size;
+}
+
+size_t ZMarkStackSpace::shrink_space() {
+  // Shrink to what is currently used
+  const size_t old_size = size();
+  const size_t new_size = align_up(used(), ZMarkStackSpaceExpandSize);
+  const size_t shrink_size = old_size - new_size;
+
+  if (shrink_size > 0) {
+    // Shrink
+    log_debug(gc, marking)("Shrinking mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
+                           old_size / M, new_size / M);
+
+    const uintptr_t shrink_start = _end - shrink_size;
+    os::uncommit_memory((char*)shrink_start, shrink_size, false /* executable */);
+  }
+
+  return shrink_size;
+}
+
 uintptr_t ZMarkStackSpace::alloc_space(size_t size) {
   uintptr_t top = Atomic::load(&_top);
 
@@ -91,23 +142,8 @@ uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) {
     return addr;
   }
 
-  // Check expansion limit
-  const size_t expand_size = ZMarkStackSpaceExpandSize;
-  const size_t old_size = _end - _start;
-  const size_t new_size = old_size + expand_size;
-  if (new_size > ZMarkStackSpaceLimit) {
-    // Expansion limit reached. This is a fatal error since we
-    // currently can't recover from running out of mark stack space.
-    fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
-          "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
-          ZMarkStackSpaceLimit / M);
-  }
-
-  log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
-                         old_size / M, new_size / M);
-
   // Expand
-  os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
+  const size_t expand_size = expand_space();
 
   // Increment top before end to make sure another
   // thread can't steal out newly expanded space.
@@ -118,6 +154,8 @@ uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) {
 }
 
 uintptr_t ZMarkStackSpace::alloc(size_t size) {
+  assert(size <= ZMarkStackSpaceExpandSize, "Invalid size");
+
   const uintptr_t addr = alloc_space(size);
   if (addr != 0) {
     return addr;
@@ -126,26 +164,21 @@ uintptr_t ZMarkStackSpace::alloc(size_t size) {
   return expand_and_alloc_space(size);
 }
 
+void ZMarkStackSpace::free() {
+  _end -= shrink_space();
+  _top = _start;
+}
+
 ZMarkStackAllocator::ZMarkStackAllocator() :
     _freelist(),
-    _space() {
-  // Prime free list to avoid an immediate space
-  // expansion when marking starts.
-  if (_space.is_initialized()) {
-    prime_freelist();
-  }
-}
+    _space() {}
 
 bool ZMarkStackAllocator::is_initialized() const {
   return _space.is_initialized();
 }
 
-void ZMarkStackAllocator::prime_freelist() {
-  for (size_t size = 0; size < ZMarkStackSpaceExpandSize; size += ZMarkStackMagazineSize) {
-    const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
-    ZMarkStackMagazine* const magazine = create_magazine_from_space(addr, ZMarkStackMagazineSize);
-    free_magazine(magazine);
-  }
+size_t ZMarkStackAllocator::size() const {
+  return _space.size();
 }
 
 ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) {
@@ -181,3 +214,8 @@ ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {
 void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) {
   _freelist.push(magazine);
 }
+
+void ZMarkStackAllocator::free() {
+  _freelist.clear();
+  _space.free();
+}

src/hotspot/share/gc/z/zMarkStackAllocator.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,7 +35,10 @@ private:
   volatile uintptr_t _top;
   volatile uintptr_t _end;
 
-  void expand();
+  size_t used() const;
+
+  size_t expand_space();
+  size_t shrink_space();
 
   uintptr_t alloc_space(size_t size);
   uintptr_t expand_and_alloc_space(size_t size);
@@ -45,7 +48,10 @@ public:
 
   bool is_initialized() const;
 
+  size_t size() const;
+
   uintptr_t alloc(size_t size);
+  void free();
 };
 
 class ZMarkStackAllocator {
@@ -53,7 +59,6 @@ private:
   ZCACHE_ALIGNED ZMarkStackMagazineList _freelist;
   ZCACHE_ALIGNED ZMarkStackSpace        _space;
 
-  void prime_freelist();
   ZMarkStackMagazine* create_magazine_from_space(uintptr_t addr, size_t size);
 
 public:
@@ -61,8 +66,12 @@ public:
 
   bool is_initialized() const;
 
+  size_t size() const;
+
   ZMarkStackMagazine* alloc_magazine();
   void free_magazine(ZMarkStackMagazine* magazine);
+
+  void free();
 };
 
 #endif // SHARE_GC_Z_ZMARKSTACKALLOCATOR_HPP

src/hotspot/share/gc/z/zMarkStackEntry.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,18 +35,22 @@
 //  ------------
 //
 //   6
-//   3                                                                  3 2 1 0
-//  +--------------------------------------------------------------------+-+-+-+
-//  |11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111|1|1|1|
-//  +--------------------------------------------------------------------+-+-+-+
-//  |                                                                    | | |
-//  |                                            2-2 Follow Flag (1-bit) * | |
-//  |                                                                      | |
-//  |                                       1-1 Partial Array Flag (1-bit) * |
-//  |                                                                        |
-//  |                                                 0-0 Final Flag (1-bit) *
+//   3                                                                5 4 3 2 1 0
+//  +------------------------------------------------------------------+-+-+-+-+-+
+//  |11111111 11111111 11111111 11111111 11111111 11111111 11111111 111|1|1|1|1|1|
+//  +------------------------------------------------------------------+-+-+-+-+-+
+//  |                                                                  | | | | |
+//  |                                            4-4 Mark Flag (1-bit) * | | | |
+//  |                                                                    | | | |
+//  |                                    3-3 Increment Live Flag (1-bit) * | | |
+//  |                                                                      | | |
+//  |                                              2-2 Follow Flag (1-bit) * | |
+//  |                                                                        | |
+//  |                                         1-1 Partial Array Flag (1-bit) * |
+//  |                                                                          |
+//  |                                                   0-0 Final Flag (1-bit) *
 //  |
-//  * 63-3 Object Address (61-bits)
+//  * 63-5 Object Address (59-bits)
 //
 //
 //  Partial array entry
@@ -72,7 +76,9 @@ private:
   typedef ZBitField<uint64_t, bool,      0,  1>  field_finalizable;
   typedef ZBitField<uint64_t, bool,      1,  1>  field_partial_array;
   typedef ZBitField<uint64_t, bool,      2,  1>  field_follow;
-  typedef ZBitField<uint64_t, uintptr_t, 3,  61> field_object_address;
+  typedef ZBitField<uint64_t, bool,      3,  1>  field_inc_live;
+  typedef ZBitField<uint64_t, bool,      4,  1>  field_mark;
+  typedef ZBitField<uint64_t, uintptr_t, 5,  59> field_object_address;
   typedef ZBitField<uint64_t, size_t,    2,  30> field_partial_array_length;
   typedef ZBitField<uint64_t, size_t,    32, 32> field_partial_array_offset;
 
@@ -86,8 +92,10 @@ public:
     // what _entry is initialized to.
   }
 
-  ZMarkStackEntry(uintptr_t object_address, bool follow, bool finalizable) :
+  ZMarkStackEntry(uintptr_t object_address, bool mark, bool inc_live, bool follow, bool finalizable) :
       _entry(field_object_address::encode(object_address) |
+             field_mark::encode(mark) |
+             field_inc_live::encode(inc_live) |
              field_follow::encode(follow) |
              field_partial_array::encode(false) |
              field_finalizable::encode(finalizable)) {}
@@ -118,6 +126,14 @@ public:
     return field_follow::decode(_entry);
   }
 
+  bool inc_live() const {
+    return field_inc_live::decode(_entry);
+  }
+
+  bool mark() const {
+    return field_mark::decode(_entry);
+  }
+
   uintptr_t object_address() const {
     return field_object_address::decode(_entry);
   }

src/hotspot/share/gc/z/zPage.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -92,6 +92,7 @@ public:
   bool is_in(uintptr_t addr) const;
 
   bool is_marked() const;
+  template <bool finalizable> bool is_object_marked(uintptr_t addr) const;
   bool is_object_live(uintptr_t addr) const;
   bool is_object_strongly_live(uintptr_t addr) const;
   bool mark_object(uintptr_t addr, bool finalizable, bool& inc_live);

src/hotspot/share/gc/z/zPage.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -171,15 +171,22 @@ inline bool ZPage::is_marked() const {
 }
 
 inline bool ZPage::is_object_marked(uintptr_t addr) const {
+  assert(is_relocatable(), "Invalid page state");
   const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
   return _livemap.get(index);
 }
 
 inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const {
+  assert(is_relocatable(), "Invalid page state");
   const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
   return _livemap.get(index + 1);
 }
 
+template <bool finalizable>
+inline bool ZPage::is_object_marked(uintptr_t addr) const {
+  return finalizable ? is_object_marked(addr) : is_object_strongly_marked(addr);
+}
+
 inline bool ZPage::is_object_live(uintptr_t addr) const {
   return is_allocating() || is_object_marked(addr);
 }

src/hotspot/share/gc/z/zStat.cpp

@@ -1125,6 +1125,7 @@ size_t ZStatMark::_nproactiveflush;
 size_t ZStatMark::_nterminateflush;
 size_t ZStatMark::_ntrycomplete;
 size_t ZStatMark::_ncontinue;
+size_t ZStatMark::_mark_stack_usage;
 
 void ZStatMark::set_at_mark_start(size_t nstripes) {
   _nstripes = nstripes;
@@ -1140,6 +1141,10 @@ void ZStatMark::set_at_mark_end(size_t nproactiveflush,
   _ncontinue = ncontinue;
 }
 
+void ZStatMark::set_at_mark_free(size_t mark_stack_usage) {
+  _mark_stack_usage = mark_stack_usage;
+}
+
 void ZStatMark::print() {
   log_info(gc, marking)("Mark: "
                         SIZE_FORMAT " stripe(s), "
@@ -1152,6 +1157,8 @@ void ZStatMark::print() {
                         _nterminateflush,
                         _ntrycomplete,
                         _ncontinue);
+
+  log_info(gc, marking)("Mark Stack Usage: " SIZE_FORMAT "M", _mark_stack_usage / M);
 }
 
 //

src/hotspot/share/gc/z/zStat.hpp

@@ -407,6 +407,7 @@ private:
   static size_t _nterminateflush;
   static size_t _ntrycomplete;
   static size_t _ncontinue;
+  static size_t _mark_stack_usage;
 
 public:
   static void set_at_mark_start(size_t nstripes);
@@ -414,6 +415,7 @@ public:
                               size_t nterminateflush,
                               size_t ntrycomplete,
                               size_t ncontinue);
+  static void set_at_mark_free(size_t mark_stack_usage);
 
   static void print();
 };