8261759: ZGC: ZWorker Threads Continue Marking After System.exit() called

Reviewed-by: sjohanss, ayang

src/hotspot/share/gc/z/zAbort.cpp

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zAbort.hpp"
+#include "runtime/atomic.hpp"
+
+volatile bool ZAbort::_should_abort = false;
+
+void ZAbort::abort() {
+  Atomic::release_store_fence(&_should_abort, true);
+}

src/hotspot/share/gc/z/zAbort.hpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZABORT_HPP
+#define SHARE_GC_Z_ZABORT_HPP
+
+#include "memory/allocation.hpp"
+
+class ZAbort : public AllStatic {
+private:
+  static volatile bool _should_abort;
+
+public:
+  static bool should_abort();
+  static void abort();
+};
+
+#endif // SHARE_GC_Z_ZABORT_HPP

src/hotspot/share/gc/z/zAbort.inline.hpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZABORT_INLINE_HPP
+#define SHARE_GC_Z_ZABORT_INLINE_HPP
+
+#include "gc/z/zAbort.hpp"
+#include "runtime/atomic.hpp"
+
+inline bool ZAbort::should_abort() {
+  return Atomic::load_acquire(&_should_abort);
+}
+
+#endif // SHARE_GC_Z_ZABORT_INLINE_HPP

src/hotspot/share/gc/z/zDriver.cpp

@@ -26,6 +26,7 @@
 #include "gc/shared/gcLocker.hpp"
 #include "gc/shared/gcVMOperations.hpp"
 #include "gc/shared/isGCActiveMark.hpp"
+#include "gc/z/zAbort.inline.hpp"
 #include "gc/z/zBreakpoint.hpp"
 #include "gc/z/zCollectedHeap.hpp"
 #include "gc/z/zDriver.hpp"
@@ -397,6 +398,19 @@ public:
   }
 };
 
+// Macro to execute a termination check after a concurrent phase. Note
+// that it's important that the termination check comes after the call
+// to the function f, since we can't abort between pause_relocate_start()
+// and concurrent_relocate(). We need to let concurrent_relocate() call
+// abort_page() on the remaining entries in the relocation set.
+#define concurrent(f)                 \
+  do {                                \
+    concurrent_##f();                 \
+    if (should_terminate()) {         \
+      return;                         \
+    }                                 \
+  } while (false)
+
 void ZDriver::gc(GCCause::Cause cause) {
   ZDriverGCScope scope(cause);
 
@@ -404,31 +418,31 @@ void ZDriver::gc(GCCause::Cause cause) {
   pause_mark_start();
 
   // Phase 2: Concurrent Mark
-  concurrent_mark();
+  concurrent(mark);
 
   // Phase 3: Pause Mark End
   while (!pause_mark_end()) {
     // Phase 3.5: Concurrent Mark Continue
-    concurrent_mark_continue();
+    concurrent(mark_continue);
   }
 
   // Phase 4: Concurrent Process Non-Strong References
-  concurrent_process_non_strong_references();
+  concurrent(process_non_strong_references);
 
   // Phase 5: Concurrent Reset Relocation Set
-  concurrent_reset_relocation_set();
+  concurrent(reset_relocation_set);
 
   // Phase 6: Pause Verify
   pause_verify();
 
   // Phase 7: Concurrent Select Relocation Set
-  concurrent_select_relocation_set();
+  concurrent(select_relocation_set);
 
   // Phase 8: Pause Relocate Start
   pause_relocate_start();
 
   // Phase 9: Concurrent Relocate
-  concurrent_relocate();
+  concurrent(relocate);
 }
 
 void ZDriver::run_service() {
@@ -456,5 +470,6 @@ void ZDriver::run_service() {
 }
 
 void ZDriver::stop_service() {
+  ZAbort::abort();
   _gc_cycle_port.send_async(GCCause::_no_gc);
 }

src/hotspot/share/gc/z/zForwarding.cpp

@@ -58,7 +58,8 @@ bool ZForwarding::retain_page() {
 
     if (ref_count < 0) {
       // Claimed
-      wait_page_released();
+      const bool success = wait_page_released();
+      assert(success, "Should always succeed");
       return false;
     }
 
@@ -129,14 +130,24 @@ void ZForwarding::release_page() {
   }
 }
 
-void ZForwarding::wait_page_released() const {
+bool ZForwarding::wait_page_released() const {
   if (Atomic::load_acquire(&_ref_count) != 0) {
-    ZStatTimer timer(ZCriticalPhaseRelocationStall);
     ZLocker<ZConditionLock> locker(&_ref_lock);
+    if (_ref_abort) {
+      return false;
+    }
+
+    ZStatTimer timer(ZCriticalPhaseRelocationStall);
     while (Atomic::load_acquire(&_ref_count) != 0) {
+      if (_ref_abort) {
+        return false;
+      }
+
       _ref_lock.wait();
     }
   }
+
+  return true;
 }
 
 ZPage* ZForwarding::detach_page() {
@@ -154,6 +165,14 @@ ZPage* ZForwarding::detach_page() {
   return page;
 }
 
+void ZForwarding::abort_page() {
+  ZLocker<ZConditionLock> locker(&_ref_lock);
+  assert(Atomic::load(&_ref_count) > 0, "Invalid state");
+  assert(!_ref_abort, "Invalid state");
+  _ref_abort = true;
+  _ref_lock.notify_all();
+}
+
 void ZForwarding::verify() const {
   guarantee(_ref_count != 0, "Invalid reference count");
   guarantee(_page != NULL, "Invalid page");

src/hotspot/share/gc/z/zForwarding.hpp

@@ -48,6 +48,7 @@ private:
   ZPage*                 _page;
   mutable ZConditionLock _ref_lock;
   volatile int32_t       _ref_count;
+  bool                   _ref_abort;
   bool                   _in_place;
 
   ZForwardingEntry* entries() const;
@@ -70,8 +71,9 @@ public:
   bool retain_page();
   ZPage* claim_page();
   void release_page();
-  void wait_page_released() const;
+  bool wait_page_released() const;
   ZPage* detach_page();
+  void abort_page();
 
   void set_in_place();
   bool in_place() const;

src/hotspot/share/gc/z/zForwarding.inline.hpp

@@ -59,6 +59,7 @@ inline ZForwarding::ZForwarding(ZPage* page, size_t nentries) :
     _page(page),
     _ref_lock(),
     _ref_count(1),
+    _ref_abort(false),
     _in_place(false) {}
 
 inline uint8_t ZForwarding::type() const {

src/hotspot/share/gc/z/zMark.cpp

@@ -26,6 +26,7 @@
 #include "classfile/classLoaderDataGraph.hpp"
 #include "code/nmethod.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/z/zAbort.inline.hpp"
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zLock.inline.hpp"
@@ -346,7 +347,7 @@ bool ZMark::drain(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, ZMarkCach
   }
 
   // Success
-  return true;
+  return !timeout->has_expired();
 }
 
 bool ZMark::try_steal_local(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks) {
@@ -497,7 +498,8 @@ bool ZMark::try_terminate() {
 class ZMarkNoTimeout : public StackObj {
 public:
   bool has_expired() {
-    return false;
+    // No timeout, but check for signal to abort
+    return ZAbort::should_abort();
   }
 };
 
@@ -506,7 +508,10 @@ void ZMark::work_without_timeout(ZMarkCache* cache, ZMarkStripe* stripe, ZMarkTh
   ZMarkNoTimeout no_timeout;
 
   for (;;) {
-    drain(stripe, stacks, cache, &no_timeout);
+    if (!drain(stripe, stacks, cache, &no_timeout)) {
+      // Abort
+      break;
+    }
 
     if (try_steal(stripe, stacks)) {
       // Stole work

src/hotspot/share/gc/z/zRelocate.cpp

@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAbort.inline.hpp"
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zForwarding.inline.hpp"
@@ -103,9 +104,14 @@ uintptr_t ZRelocate::relocate_object(ZForwarding* forwarding, uintptr_t from_add
       return to_addr;
     }
 
-    // Failed to relocate object. Wait for a worker thread to
-    // complete relocation of this page, and then forward object.
-    forwarding->wait_page_released();
+    // Failed to relocate object. Wait for a worker thread to complete
+    // relocation of this page, and then forward the object. If the GC
+    // aborts the relocation phase before the page has been relocated,
+    // then wait return false and we just forward the object in-place.
+    if (!forwarding->wait_page_released()) {
+      // Forward object in-place
+      return forwarding_insert(forwarding, from_addr, from_addr, &cursor);
+    }
   }
 
   // Forward object
@@ -339,8 +345,15 @@ public:
   }
 
   void do_forwarding(ZForwarding* forwarding) {
-    // Relocate objects
     _forwarding = forwarding;
+
+    // Check if we should abort
+    if (ZAbort::should_abort()) {
+      _forwarding->abort_page();
+      return;
+    }
+
+    // Relocate objects
     _forwarding->object_iterate(this);
 
     // Verify

src/hotspot/share/gc/z/zStat.cpp

@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAbort.inline.hpp"
 #include "gc/z/zCollectedHeap.hpp"
 #include "gc/z/zCPU.inline.hpp"
 #include "gc/z/zGlobals.hpp"
@@ -642,6 +643,12 @@ void ZStatPhaseCycle::register_start(const Ticks& start) const {
 }
 
 void ZStatPhaseCycle::register_end(const Ticks& start, const Ticks& end) const {
+  if (ZAbort::should_abort()) {
+    log_info(gc)("Garbage Collection (%s) Aborted",
+                 GCCause::to_string(ZCollectedHeap::heap()->gc_cause()));
+    return;
+  }
+
   timer()->register_gc_end(end);
 
   ZCollectedHeap::heap()->print_heap_after_gc();
@@ -712,6 +719,10 @@ void ZStatPhaseConcurrent::register_start(const Ticks& start) const {
 }
 
 void ZStatPhaseConcurrent::register_end(const Ticks& start, const Ticks& end) const {
+  if (ZAbort::should_abort()) {
+    return;
+  }
+
   timer()->register_gc_concurrent_end(end);
 
   const Tickspan duration = end - start;
@@ -730,6 +741,10 @@ void ZStatSubPhase::register_start(const Ticks& start) const {
 }
 
 void ZStatSubPhase::register_end(const Ticks& start, const Ticks& end) const {
+  if (ZAbort::should_abort()) {
+    return;
+  }
+
   ZTracer::tracer()->report_thread_phase(name(), start, end);
 
   const Tickspan duration = end - start;